{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "作业要求：\n",
    "\n",
    "独立调用xgboost或在sklearn框架下调用均可。\n",
    "\n",
    "1. 模型训练：超参数调优\n",
    "\n",
    "a) 初步确定弱学习器数目： 20分\n",
    "\n",
    "b) 对树的最大深度（可选）和min_children_weight进行调优（可选）：20分\n",
    "\n",
    "c) 对正则参数进行调优：20分\n",
    "\n",
    "d) 重新调整弱学习器数目：10分\n",
    "\n",
    "e) 行列重采样参数调整：10分\n",
    "\n",
    "2. 调用模型进行测试10分\n",
    "\n",
    "3. 生成测试结果文件10分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#首先import必要的模块\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "#dpath = './data/'\n",
    "train = pd.read_csv(\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level'] \n",
    "\n",
    "train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#直接调用xgboost内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  #调用xgboost自带的cv函数，可以方便的确定弱学习器的数目或者迭代次数\n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    print('n_estimators: %s',n_estimators)\n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    #train_predprob = alg.predict_proba(X_train)\n",
    "    #logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "   #Print model report:\n",
    "   # print (\"logloss of train :\" )\n",
    "   # print logloss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: %s 270\n"
     ]
    }
   ],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "得到最佳的n_estimators,下面用图来现实"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcHHW57/HP0z37nmQm+84eEFnCroAgsimgIIIHj6BXjh65Lkf0wNXj4XLF7XA4KnD1IgdwQRBxQwVUdlRAAoQlLCErmezb7Pv0c//41Uw6k56kk0xPzUx/369Xvaa7qrr6qa6kv12/qvqVuTsiIiIAibgLEBGRkUOhICIi/RQKIiLST6EgIiL9FAoiItJPoSAiIv0UCiJpzOx/mdmtcdchEheFwihjZhVmtsLMPpw2rtLM3jKzC9LGzTez35vZVjNrMLNXzew6MxsXTb/UzHrNrCUalpnZp3Jc+8lmVp/L99gdmepx96+7+//I0futMLN352LZuTBc22u0fS5jnUJhlHH3FuBy4LtmVheN/jawwN3vBTCz44HHgL8CB7p7DXAG0AO8PW1xT7l7hbtXABcA3zazw4dnTWR3mFlB3DVInnB3DaNwAO4A7gJOBjYDU9Km/QW4cRevvxT4y4Bxfwc+nPb8HGAR0EAImYPSph0UjWuI5jknbdpZwKtAM7AauBIoB9qBFNASDVMHWa+bgT9Er38G2CeLz+NA4M/AFuAN4MI9qQe4Bvhp9LrZgAOXAauArcAngaOAl6J1vyntffYBHom2xybgTqAmmvaT6L3ao/f6Uhaf8QrgX6P36gQKouero3V5Azg1w2dxLLAOSKaNez/wUvT4aGAB0ASsB24Y5DM9GagfZFo18GNgI7AS+AqQiKYlgf+MPoPlwBXR51gwyLJWAO8eZNongCXRdr2v798MYMB/ARuAxugzOmSw7R33/9fRNMRegIY93HAwDlgb/ce7LG18OdALnLyL119KWihEX3QNwP7R8/2BVuA0oBD4UvSfsyh6vgT4X9HzU6L/gAdEr10LvDOtziOix4N+yaTVcUf0BXB09CV4J3D3Ll5TTvjSvix6zRHR53Lw7tZD5lD4AVACvAfoAH4DTASmRV9KJ0Xz7xt9XsVAHfAE8J20ZW/35bezzzht/oXADKAUOCBaz6lp9WUMTGApcFra818AV0WPnwI+Ej2uAI4dZBmDbi9CIPwWqIzqWAx8PJr2ScKX8vTo836IPQiF6N/Vpmh7FgM3Ak9E004HngNqCAFxENEPo8G2t4bsBjUfjVLuvpXwC7MM+FXapHGEZsF1fSPM7NvRcYVWM/tK2rzHRuNbCHsJPwHejKZ9CPiDu//Z3buB6wlfTMcTfolWAN909y53fwT4PXBx9NpuYJ6ZVbn7Vnd/fjdX71fu/nd37yGEwmG7mP+9wAp3v93de6L3+yWhSWwo6vk/7t7h7n8ifInf5e4b3H018CRwOIC7L4k+r0533wjcAJy0k+Xu7DPu8z13X+Xu7YSwL47WpdDdV7j70kGWfRfR9jCzSsKv57vSPo99zazW3Vvc/end+TDMLBnVfrW7N7v7CsKewUeiWS4Evuvu9dG/02/uzvLT/ANwm7s/7+6dwNXAcWY2O1qHSsIeorn7a+6+Nm399mZ75zWFwihlZpcQfqE9BHwrbdJWQjPFlL4R7v4lD8cVfk34Jd3naXev8XBMYTJwMPD1aNpUQrNA3zJShF+p06Jpq6JxfVZG0wDOJ3wJrTSzx83suN1cvXVpj9sIAbQzs4BjooBrMLMGwhfK5CGqZ33a4/YMzysAzGyimd1tZqvNrAn4KVC7k+Xu7DPusypt+hLgc4S9mQ3Re00dZNk/Az5gZsXAB4Dn3b3vvT5O2Et53cyeNbP37qTGTGoJe4gr08alb/+p6XUPeLw7Bn4+LYSmuWnRD5GbCE2N683sFjOrimbd2+2d1xQKo5CZTSS0p34C+CfgQjM7EcDdWwnt8B/YnWW6+3rCr+v3RaPWEL5s+97TCM0Yq6NpM8ws/d/PzGga7v6su59LaGL5DXBP39vsTk27YRXweBRwfUOFu39qmOv5RrTMQ929CriE0LTRZ+D77ewzzvgad/+Zu78jep2z/Q+C9PleJXyhngl8mBASfdPedPeLCZ/Ht4B7zaw8+9VkE+HX+Ky0cf3bn9B8Mz1t2ozdWHa6gZ9POTCBbf/OvufuRxJ+zOwPfDEaP9j2liwoFEanm4DfuPuj0S7zl4AfRr8KiZ5/zMyuigIEM5sOzBlsgWY2gXAwclE06h7gbDM71cwKgS8QDnb+jRA6rcCXzKzQzE4mhMndZlZkZv9gZtVRk0gTodkDwi/sCWZWPUSfQ5/fA/ub2UeiegrN7CgzO2iY66kkHERuMLNpRF9SadYDc9Oe7+wz3oGZHWBmp0TbuYOwl9Kbad7Iz4DPACcSjin0LecSM6uL9kwaotGDLsfMStIHwp7oPcB10enQs4B/IewZ9a3XZ81smpnVEA6O70rhgPcpiOq/zMwOi9b568Az7r4i2r7HRJ9ba/R59O5ie0s24j6ooWH3BuA8wi+omgHjHwauS3t+DHA/4T99A/AKcB0wIZp+KeE/S9+ZNxsIbc4T05bxfsIBw0bgcaIDt9G0g6NxjdE874/GFwEPEpqxmoBngXekve42QhNAA4OfffS1tOcns4uD09F8BxDOWNoYLf8RwrGI3aqHzAeaC9LmryftID7hi/AraZ/Jc9HnuZDwJV+fNu+5wFvRe12ZxWe8gu0PTB9KOPbTTDgY//tMn2Ha/DMJX+B/GDD+p9H2biH8CDhvkNefHK3/wGFfwrGrn0af9yrgq2w7+6iAsCe7mXD20ecJexY2yPusyPAeX4umfZJw0LxvfadH408lnHHUwrYzvSp2tb017Hqw6AMWEckJMzsT+IG7z9rlzBI7NR+JyJAys1IzO8vMCqJmtH8nnOQgo4D2FGRUMLN3Ag9kmubh7CkZIcysjNAUdiDhuMcfgM+6e1OshUlWFAoiItJPzUciItJv1HWyVVtb67Nnz467DBGRUeW5557b5O51u5pv1IXC7NmzWbBgQdxliIiMKma2ctdzqflIRETSKBRERKSfQkFERPopFEREpJ9CQURE+ikURESkn0JBRET65U0obGzu5LkVm1G3HiIig8ubULjnu1/k8Nv3oaO9Pe5SRERGrLwJhaP3n07CnOaGTXGXIiIyYuVNKCTLawBobdoScyUiIiNX3oRCUfl4ANqbNsdciYjIyJU3oVBcMQ6AzhbtKYiIDCZvQqGsagIA3a1bY65ERGTkyp9QqA6h0KNQEBEZVN6EQmVNCAVvb4y5EhGRkStvQqGguIxOL4QOhYKIyGDyJhQAmq2cZJdCQURkMHkVCm2JCgq6muIuQ0RkxMqrUGhPVlLUrVAQERlMzkLBzG4zsw1m9sog083MvmdmS8zsJTM7Ile19OkqqKSktznXbyMiMmrlck/hDuCMnUw/E9gvGi4Hvp/DWgDoLqyiNNWa67cRERm1chYK7v4EsLPLh88FfuzB00CNmU3JVT0AvcVVVKRacvkWIiKjWpzHFKYBq9Ke10fjdmBml5vZAjNbsHHjxj1+Qy+uppJWenp693gZIiJjWZyhYBnGZbwDjrvf4u7z3X1+XV3dnr9haTUFlqK5WaeliohkEmco1AMz0p5PB9bk8g0TpaFTvBbdU0FEJKM4Q+E+4B+js5COBRrdfW0u37Bo2UMAtKr7bBGRjApytWAzuws4Gag1s3rg34FCAHf/AXA/cBawBGgDLstVLX2K3vEpePBJ2hv3/LiEiMhYlrNQcPeLdzHdgU/n6v0zKa8OxyO6mhUKIiKZ5NUVzRXjJgHQ06wb7YiIZJJXoVA5LuwppNoUCiIimeRVKCSLy2inCGtXKIiIZJJXoQDQbJUkOxviLkNEZETKu1BoTVZT1KVQEBHJJO9CoaOgmtJuXdEsIpJJ3oVCV1EN5SndU0FEJJO8C4We4hoqFQoiIhnlXSh46XiqaaGjqzvuUkRERpy8CwUrG0/SnIat6hRPRGSgvAuFworxADRv2RBzJSIiI0/+hUJlLQBtjdpTEBEZKO9CobxmIgDtjdpTEBEZKO9CoXLCZAC6mtRTqojIQHkXCtUTpgKQalEoiIgMlHehkCippIMirFXNRyIiA+VdKGBGo1VT2KFbcoqIDJR/oQC0FoyjuFPdZ4uIDJSXodBeNJ6Knq1xlyEiMuLkZSh0l0ygKqWeUkVEBsrLUEiV1TGeRto61f+RiEi6vAyFRGUdxdbD5s062Cwiki4vQ6GoahIATZtXx1yJiMjIkpehUFoTrmpu27Iu5kpEREaWvAyF8glTAOhoXB9zJSIiI0tehkJNbejqokehICKynZyGgpmdYWZvmNkSM7sqw/RZZvawmb1kZo+Z2fRc1tOnqGoiKQxvUSiIiKTLWSiYWRK4GTgTmAdcbGbzBsx2PfBjdz8UuBb4Rq7q2U6ygAarJtmmUBARSZfLPYWjgSXuvszdu4C7gXMHzDMPeDh6/GiG6TnTVFhLWYd6ShURSZfLUJgGrEp7Xh+NS/cicH70+P1ApZlNGLggM7vczBaY2YKNG4fmi7y9qJbKbt19TUQkXS5DwTKM8wHPrwROMrMXgJOA1UDPDi9yv8Xd57v7/Lq6uiEprqdsEuN9Kz29qSFZnojIWFCQw2XXAzPSnk8H1qTP4O5rgA8AmFkFcL67D0unRF45mdr1jWxsbmNSTcVwvKWIyIiXyz2FZ4H9zGyOmRUBFwH3pc9gZrVm1lfD1cBtOaxnO4XVU0mYs3m9rmoWEemTs1Bw9x7gCuCPwGvAPe6+yMyuNbNzotlOBt4ws8XAJOC6XNUzUOmEcHijeeOqXcwpIpI/ctl8hLvfD9w/YNxX0x7fC9ybyxoGU1kXLolo31Ifx9uLiIxIeXlFM0B1XTjc0d2wNuZKRERGjrwNhYKqyeGq5mZ1iici0idvQ4FkAQ2JGgpbFQoiIn3yNxSApsJJVHQqFERE+uR1KHSUTWZ8zwbcB15TJyKSn/I6FHorpzGZzWxp6Yy7FBGRESGvQyE5bgbl1sm6DeotVUQE8jwUympnArB17fKYKxERGRnyOhSqJ88FoH3jypgrEREZGfI6FKomzQKgp0FdXYiIQJ6HglVMotuTrF25JO5SRERGhJz2fTTiJZI0FNQxt2Br3JWIiIwIeb2nANBSMpkqXcAmIgIoFOiqnMEU30BTR3fcpYiIxC7vQyExbhaT2Er9RjUhiYjkfSiUTpxLwpxN9TrYLCKS96FQM20/AFrWLY25EhGR+OV9KFRM3heA7s0r4i1ERGQEyPtQoGIyXRRS0PhW3JWIiMROoZBIsIY6ko3q6kJERKEAePVMprGBrp5U3KWIiMRKoQCkxs1hlq1n1ZbWuEsREYmVQgEonrQfVdbGqvr6uEsREYnVLkPBzPYxs+Lo8clm9hkzq8l9acNn3Ix5ADTWvxpzJSIi8cpmT+GXQK+Z7Qv8NzAH+FlOqxpm5VMPAKBr/eKYKxERiVc2oZBy9x7g/cB33P3zwJTcljXMqmfSQ5KCBt2BTUTyWzah0G1mFwMfBX4fjSvMXUkxSBawpWgaVW06LVVE8ls2oXAZcBxwnbsvN7M5wE+zWbiZnWFmb5jZEjO7KsP0mWb2qJm9YGYvmdlZu1f+0GmvnMXU3jVsbe2KqwQRkdjtMhTc/VV3/4y732Vm44BKd//mrl5nZkngZuBMYB5wsZnNGzDbV4B73P1w4CLg/+72GgyRROsGZts6Fq9rjKsEEZHYZXP20WNmVmVm44EXgdvN7IYsln00sMTdl7l7F3A3cO6AeRyoih5XA2uyL31oVb7jnyi1Ltas1MFmEclf2TQfVbt7E/AB4HZ3PxJ4dxavmwasSnteH41Ldw1wiZnVA/cD/zPTgszscjNbYGYLNm7cmMVb777qmYcA0LLqlZwsX0RkNMgmFArMbApwIdsONGfDMozzAc8vBu5w9+nAWcBPzGyHmtz9Fnef7+7z6+rqdqOE7FldOC01sen1nCxfRGQ0yCYUrgX+CCx192fNbC7wZhavqwdmpD2fzo7NQx8H7gFw96eAEqA2i2UPvdIaGgvrqGpeivvA7BIRyQ/ZHGj+hbsf6u6fip4vc/fzs1j2s8B+ZjbHzIoIB5LvGzDPW8CpAGZ2ECEUctM+lIXFvVOZmVrFxubOuEoQEYlVNgeap5vZr81sg5mtN7Nfmtn0Xb0uuuDtCsJexmuEs4wWmdm1ZnZONNsXgE+Y2YvAXcClHuPP9OkHHM5+tppFaxriKkFEJFYFWcxzO6Fbiw9Gzy+Jxp22qxe6+/2EA8jp476a9vhV4IRsi821cWufosQ6WbX0NThwctzliIgMu2yOKdS5++3u3hMNdwC5Odobs5IPfh+A9lULY65ERCQe2YTCJjO7xMyS0XAJsDnXhcVi4jxSJCjetCjuSkREYpFNKHyMcDrqOmAtcAGh64uxp7CUhrLZTOtcSmNbd9zViIgMu2zOPnrL3c9x9zp3n+ju5xEuZBuTFnbPYF5iJS+vVncXIpJ/9vTOa/8ypFWMIMed8C6m2WZeX6ZutEUk/+xpKGS6WnlMKH31XgAalz0XcyUiIsNvT0Nh7F7ye9kfACjesFBXNotI3hn0OgUzaybzl78BpTmrKG6lNTSWz2b/psW8taWNWRPK465IRGTYDLqn4O6V7l6VYah092wuehu1fOqRHJZYymW3/T3uUkREhtWeNh+NaVX7HMNEa+DdM3rjLkVEZFgpFDJITJ8PQNeKZ2KuRERkeCkUMpn8NnoSxcxoeZkNTR1xVyMiMmwUCpkUFNEx8XDmJ97gmeVb4q5GRGTYZNN1drOZNQ0YVkXdac8djiLjULbvCRxsK1iw+K24SxERGTbZ7CncAHyRcH/l6cCVwA+Bu4HbcldavBKzjqfAUjQsfkrXK4hI3sgmFM5w9//n7s3u3uTutwBnufvPgXE5ri8+M44mhTG3/UWWbmyNuxoRkWGRTSikzOxCM0tEw4Vp08buT+iSKronHcYJiUU8sTi2O4SKiAyrbELhH4CPABui4SPAJWZWSrjd5phVvP8pHGZL+OFDuumOiOSHbLrOXubu73P32mh4n7svcfd2d//LcBQZm7knU2ApDu5+hY3NnXFXIyKSc9mcfTQ9OtNog5mtN7Nfmtn04SgudtOPJpUs5gR7mQdfWRt3NSIiOZdN89HtwH3AVMIZSL+Lxo19hSUk9nkXpyWf59t/fD3uakREci6bUKhz99vdvSca7gDqclzXyHHge5luG5nZtZT1urpZRMa4bEJhk5ldYmbJaLgE2JzrwkaMA87CLcHpib/zwMtqQhKRsS2bUPgYcCGwDlgLXABclsuiRpTyCVhRJecmn+Z3LykURGRsy+bso7fc/Rx3r3P3ie5+HvCBYaht5DjlK8yydTS89Qpvrm+OuxoRkZzZ0w7x/mVIqxjpDjwbgLMKnuPOZ9QXkoiMXXsaCpbVTGZnmNkbZrbEzK7KMP2/zGxhNCw2s4Y9rCe3qqfBtCM5M/l3fvLUStq6euKuSEQkJ/Y0FHbZvYWZJYGbgTOBecDFZjZvu4W4f97dD3P3w4AbgV/tYT25d8j5zGMZc1nFfQvXxF2NiEhODBoKg3SZ3WRmzYRrFnblaGBJdEV0F6FX1XN3Mv/FwF27Vf1wOvRDeKKQT1b+jZ8+s1I9p4rImDRoKLh7pbtXZRgq3b0gi2VPA1alPa+Pxu3AzGYBc4BHdqf4YVVeix1wJqd2PcIbq7fw7IqtcVckIjLkcnnntUzHHQb7eX0RcK+792ZckNnlZrbAzBZs3Bhjj6WHf4Qamjiv7CVuenRJfHWIiORILkOhHpiR9nw6MFhj/EXspOnI3W9x9/nuPr+uLsaLqfc9FSqnckXNUzyxeCNnfffJ+GoREcmBXIbCs8B+ZjbHzIoIX/z3DZzJzA4g3KznqRzWMjQSSTjsYmZufYpDKpooKkiQSunYgoiMHTkLBXfvIdxv4Y/Aa8A97r7IzK41s3PSZr0YuNtHy5HbIy/FMP5z5lMsXNXAaf/1eNwViYgMmWwOGO8xd78fuH/AuK8OeH5NLmsYcjUz4eDz2H/xrzh26tksb+mhtbOH8uKcfpQiIsMil81HY9fxn8G6mrmh8xrWN3Xy7hu0tyAiY4NCYU9MPQwOfC9Tu1fx8SMqWd/UwXMrt8RdlYjIXlMo7KlT/g26mvnXVf9MYTLBh3/4jLq/EJFRT6GwpyYeCG//MEVtG7jrQzPo7Elx8n88FndVIiJ7RaGwN06+Cnq7OOLB85hSXcKG5k5+pl5URWQUUyjsjXGz4J1fgNaN/OXCJNWlhXz51y/z9LL8uTGdiIwtCoW9deKVUFBC8q4P8eQXjqO4MMEltz6jm/GIyKikUNhbhaVw0c+gp4OqZ2/kj587ETM463tPsnxTa9zViYjsFoXCUNj3VHjbhfDE9cxqfYUDJ1fhDqf/1xOce9Nf4q5ORCRrCoWhcvb1UDMD7v0Yv/v4PO674h2k3Fm0pokFK3QNg4iMDgqFoVJSDRfcDi3r4Tf/zLzJFTz0LydRkDAu+MFTnHL9Y7oxj4iMeAqFoTTtCHjP12DxA/Dk9cyuLeepq0+lsqSAZZtamf+1h2hs7467ShGRQSkUhtox/wSHXgSPXgev/4Fx5UUs/Op7uPI9+7O5tYv5X/szv36hXnsNIjIiKRSGmhm87zsw9Qj41eWwZiHJhHHFKfvx20+fQFEywed//iJvu+ZPvFTfEHe1IiLbUSjkQmEpXHQnlI6DH58Dq58H4O0zanj5mtOZU1tOR3cv59z0V4762p/Z2NwZc8EiIoGNtmaM+fPn+4IFC+IuIztbV8KP3gvtjfCRX8H0+f2Tmjq6ufHhN7n1yeU4MK2mhF9/+gQmVpbEV6+IjFlm9py7z9/lfAqFHGtYFYKhdTNcci/MPHa7yUs3tnDB9//G1rZwALq2oog7LjuaQ6ZVx1GtiIxRCoWRpHE13HgE9HTCOd+DI/5xh1mWbWzh4h8+zfqm0JRUWVLAf1zwdk6bN4lkwoa7YhEZYxQKI03bFrj3Mlj2GBx5GZzxjXDsYYDG9m7ed+OTrNrSjgPFBQlqK4q56xPHMnNC2bCXLSJjg0JhJEr1wsPXwl+/AxMPhgtuC/dlyKCnN8VDr63nS/e+RFNHuHlPeXGSfz55X04/eDL7TqwYzspFZJRTKIxkbz4Ev/kkdDbD6V+H+R8Lp7IOYnVDO79duJqbH1lCa1cvACWFCT563GyO3WcCR84aR1VJ4XBVLyKjkEJhpGteH4Jh6SNQOgGueBbKJ+zyZWsa2nnotfVc/8c3+vcgABIGEyuLueacQzhq9jgmVBTnsnoRGWUUCqNBKgVP3wx/+jdIJOH0b4S9hmRBVi9v6+rhhbcaeGb5Fm7/63KaB4REbUUxXz77II6ZM4HJ1TrVVSSfKRRGkw2vwQP/Cssfh4nz4IxvwtyTdnsxnT29vFzfyOd+vpC1jR30prZtWzNImjGpqpirzzqIAyZVMru2nMKkrl8UyQcKhdHGHV7/Pfzy4+HU1YPeFzrXGzd7jxfZ05vitbXN/H3FFm565E0a2rsZuLkTBjVlRVx6/GxmTShjxvgyZowro7aiCNvJcQ4RGV0UCqNVdwc8dSM8+vUQFO/8ApzwWSipGpLFd3T3smRDC5+9+wVWbWmj16EgYXT2pHaYN2FQVVLIeYdPY/q4UmaOj0JjfBkVxdk1cYnIyKBQGO0aV8ND/w4v/wISBXDy1XD05UMWDgO1d/VSv7WNt7a0sWpLGz94fCmbWrpIuZPK8E/ECE1SNWVFfOioGUwfV8rU6lImVZUwpbqEmrJC7WmIjCAjIhTM7Azgu0ASuNXdv5lhnguBawAHXnT3D+9smXkTCn1WPw+PfwsWPxgORr/zSjjmk1A2fthKcHca2rpDYGxt41sPvM66pg5SDin3HZqk+piFHhfNDDOYWFnCP500l3FlRYwvL6KmrJAJ5cWMLy+iqEDHNkRyKfZQMLMksBg4DagHngUudvdX0+bZD7gHOMXdt5rZRHffsLPl5l0o9FnzAjxxfTjuYIkQDEf8I0w8KO7K6E0565s6WNfUwbrGDr71wOusaWzHPQRKCgYNjnQlBQmSCaMjasrq2xuZWFnMZSfMoaqkkKrSAqpKC6kqKaQ6+ltRUqCuQER2YSSEwnHANe5+evT8agB3/0baPN8GFrv7rdkuN29Doc+G1+BH74PWjeH5tCPh8EvgkPPDLUFHsJ7eFA3t3Wxt7WJrWzdbWjv55gOvs6axA3ensqSQ3pTT0tGNE3Ydd+efZ1+IlBcXUJAwmjt6+q8JNMJEA6bWlHLlew6grDhJRXEBZUVJyooKKClMUFKQpKQwSXFBgoSCRsaQkRAKFwBnuPv/iJ5/BDjG3a9Im+c3hL2JEwhNTNe4+4MZlnU5cDnAzJkzj1y5cmVOah5VWjfBSz+HF34KG6Kdr/I6OP9WmH0iJMZOc0wq5bR09dDU3k1Tew+N7d00dXTT1N7NjY8sYV1j+3YBUlqYpCfldHT3sjf/us3CMkOgRH/DQypKCkkY2wVP37S+8JlSXYKZsbaxvX/6zPFlfPnseRQlExQVJCguSFBSmKC4IARRUUGCwmhaQcJ0XEaGzEgIhQ8Cpw8IhaPd/X+mzfN7oBu4EJgOPAkc4u6D3pIs7/cUBnIPTUsv/BSeuwO8F5LFcOyn4G0fhMmHxF3hiODudPakaOvqpbWzh7auXlo6e2jr6uHa373Kqq1t/aHiQF1FESmHTS2d0eu3TcOdkqIkqRR09PSCs1fhszN9ez/b/TdNCygIe0YGtHb1kilCxkUH/be2de2w7PDAmFhRjBlsyHDDp75QnFZTigH1De3bvz4yp7acr73/bSTNSCaMgqRte5xIkNzuue34XCGYUyMhFLJpPvoB8LS73xE9fxi4yt2fHWy5CoWd6G6H134P938ROraGcYVlcOKVISBqZsZbXx7oC5/O7hQdPb1096bo6XV6Uim6esLf7l6nqydFV2+Kzu5evv3g69Rvbd8uWPrCZ0JFMe7O5tauaPkD5gHKipK4hyvc2WEZUJhM4Dg9vZ4f/pK5AAAPbElEQVSz8BpKxvYhawMmFiUTmEFXhtOo+/bUyouSmBktnT2DBGURZuwQlOkmlIdrdTa3pAVlFFoGTKoqBoz1TR2DLmNqdQmYsTYK0kzFDAzb6TWlfP60A8KJGlEiJyycsHHg5EpmjN+z3pJHQigUEJqGTgVWEw40f9jdF6XNcwbh4PNHzawWeAE4zN03D7ZchUKWWjfBol+HU1pXPRPGFVfBqV+FA86E6unx1icjRirl9LrTmwpDT2rb4/A8xafvfL6/iW7pxpbtXt8XYLMmlOPuvLWlLXP4eGhSc2BtY8f2r4+WUVtZjKftoe34PlBdWogDTe3d278+bTmlUVC2d/dmeB8oSIZv550FZd8hpUynZMdl9oQyHvviu/botbGHQlTEWcB3CMcLbnP368zsWmCBu99nYV/xP4EzgF7gOne/e2fLVCjsgS3LQzg8eQP0RL9YJh8KB54dAmLyoTvtpVVEwl7gdqEZnY7t0d+Ue39wevQ4lT4t+qpNZZg/eofw2r5lsP1r3WFqTcked3Y5IkIhFxQKe8EdNr0Jb9wPT14fuu4GqJoewuGAM2H2O6GgKN46RWTIKRRk11o2wpt/hDceCEHhURttWW3od2nuyVA1Jc4KRWSIKBRk93S3w7LHQzi8+DPoDe21FJaF24fucwrMOh6KdEtQkdFIoSB7LpWC9a+EGwD95QboaIwmGBRXwnFXwJwTw4VzamoSGRUUCjJ0utth5d9g2aOw4Dboat02raQGjr8CZr0Dph0BBbrjm8hIpFCQ3GnbEkJi+ePhornutjDeEoBB1TQ490aYekTOenUVkd2jUJDh07oZ3vobrPgrPP9j6E7bkygsC/0yTT8Kps+H2gOyvt2oiAwdhYLEp30rrH4O6p+Dp7+/7epqiPYmgPKJ8K6rYcph4RakOjYhklMKBRk53GHzUli9ANa+FJqcOhu3n6eoAg69MITE5EOg7iCd6SQyhBQKMrK5w9bloTO/P/0bNK+Lxm/rlgBLQOm4cMe5ifNg0sHhntWJZCwli4xm2YaCGnclHmYwfm4YDjk/jEulQlCsXxS6A//7LaEp6rFvpL0uEY5TFJbDOz+/LSzKa+NZD5ExRnsKMvJ1tcHG17eFxcI7oaOJ7bpCSxaG02InHRyGifOg7kAoLImtbJGRRM1HMva1bAhBsX4R/PW70LZpW1cdfQpKYf/Ttw+Lmllj6iZEItlQKEh+SvXClmXbwuLZW0MTVPpehSVDE1RRGRz/GZiwD0zYN4SFzoKSMUqhIJKus2VbE9T6RfDi3dA5oAkKAAtXab/9QzB+n22BUT1DexcyqikURLLRtiWcLrtlKWxeAgtuh/YtgG1/JhRsOxvqyMugdj+YsB/U7gsl1bGULrI7FAoie8MdWtaHoNj0Zvi78M7QOeDA4xZ9HQUe/P60sNgv3P40WRhL+SIDKRREcqWnK5w6u+lN2PwmPPV/oW1zFBYD/j8VFENvD5TXwXH/DOPmwPg54W9xRSzlS35SKIjEoW3Ltj2LrcvDrVAXPxj1LJvh+EVxBRxw9ragGD83PC6boFukypDSxWsicSgbDzOPCcNA7Q3bgqL/7wpY9Gvo3fFG9RSWw76nhpAYP3fbXkbVNF3VLTmjUBAZLqU1UHo4TD18x2ndHdCwcvvAePkeePPP0NPBjs1SpTD7Hdv2LPoCo2aWLtiTvaJQEBkJCkug7oAw9Dnr2+FvqheaVg/Yw1gOSx6CJX/ecVnJIph+dLSHEQVGzaxw4Lu8Vs1SslMKBZGRLpEMX+g1M4GTtp/mHg5yDwyM1/8AK//KjscxCKfWllSHPqf6llszMwSHjmXkPYWCyGhmFn79l9fCjKN2nN7ZEpqlGt5KG1aG+28/+98oNGQghYLIWFZcsa3fp0w6mqBx1SChceuO8/fdJGm70JiVFhrjFRqjnEJBJJ+VVEHJzkKjERoGhMaLd4e76e0yNC6AmhlQPT10E1I9HSom6cypEU7XKYjInssUGg0rYemj29+ru5+FA+EFxbD/GVFgpIVG9fQQVDLkdJ2CiOReSTVMrg63UM2koymcOdVYH5qpGuvD8MYD8PIvyHxMIxnGl1SHrkMGhkblFHUfkkM5DQUzOwP4LpAEbnX3bw6YfinwH8DqaNRN7p5hn1RERqWSqjBMPCjz9FRv6GOqcfX2odFYD8sfCx0UZgqOvr2NfU/bMTSqp4eOC3VsY4/kLBTMLAncDJwG1APPmtl97v7qgFl/7u5X5KoOERnBEkmomhqGTGdPQegiJD000vc83rg/urhvAEuE0EgWw7xzo9CYti00qqaF6bKDXO4pHA0scfdlAGZ2N3AuMDAUREQGV1QOdfuHIZNUKtx1b+CeRuOqcGzj+R+TeW+jMITGvqduv6dRNS08z9ML/XIZCtOAVWnP64EMHcJwvpmdCCwGPu/uqwbOYGaXA5cDzJw5MwelisiolUhAxcQwTDsy8zzdHWl7GPXbH+NY/Cfoad/xNZbY1kx14Pu2b57qC4+istyuWwxyGQqZInZgXP8OuMvdO83sk8CPgFN2eJH7LcAtEM4+GupCRWSMKyyJ7qK3T+bp7qGH26YMofHmQ+FeGpn2NhIFITTmviscAK+cvO1v1TSomgLFVaNqjyOXoVAPzEh7Ph1Ykz6Du29Oe/pD4Fs5rEdEJDMzKJ8QhilvzzxPTxc0r4kCY8CB8aWPQHfbIMuOzqYqroQDzgqhUTU1Co8oQComQXJknAyayyqeBfYzszmEs4suAj6cPoOZTXH3tdHTc4DXcliPiMieKyiCcbPDMJiuNmhZB01roXktNK2J/q4O4175VeZu0iEc45g4DyqnbtvjqEoLjsqpw3LFeM5Cwd17zOwK4I+EU1Jvc/dFZnYtsMDd7wM+Y2bnAD3AFuDSXNUjIpJzRWVRd+ZzB58n1Qutm0JYNK8Lex/N66LwWBv2PJY8BKnuHV87bi589oXc1Y+uaBYRGZl6OsM1HM3roj2OdeEeGoNdKLgLuqJZRGQ0KyhO6zJ9+CSG9d1ERGREUyiIiEg/hYKIiPRTKIiISD+FgoiI9FMoiIhIP4WCiIj0UyiIiEi/UXdFs5ltBFbu4ctrgU1DWM5INNbXcayvH2gdx4KRuH6z3L1uVzONulDYG2a2IJvLvEezsb6OY339QOs4Fozm9VPzkYiI9FMoiIhIv3wLhVviLmAYjPV1HOvrB1rHsWDUrl9eHVMQEZGdy7c9BRER2QmFgoiI9MubUDCzM8zsDTNbYmZXxV3PUDCzFWb2spktNLMF0bjxZvZnM3sz+jsu7jp3h5ndZmYbzOyVtHEZ18mC70Xb9CUzOyK+yrM3yDpeY2aro2250MzOSpt2dbSOb5jZ6fFUnT0zm2Fmj5rZa2a2yMw+G40fE9txJ+s3Nrahu4/5gXCP6KXAXKAIeBGYF3ddQ7BeK4DaAeO+DVwVPb4K+Fbcde7mOp0IHAG8sqt1As4CHgAMOBZ4Ju7692IdrwGuzDDvvOjfazEwJ/p3nIx7HXaxflOAI6LHlcDiaD3GxHbcyfqNiW2YL3sKRwNL3H2Zu3cBdwPnxlxTrpwL/Ch6/CPgvBhr2W3u/gSwZcDowdbpXODHHjwN1JjZlOGpdM8Nso6DORe429073X05sITw73nEcve17v589LgZeA2YxhjZjjtZv8GMqm2YL6EwDViV9ryenW/E0cKBP5nZc2Z2eTRukruvhfCPF5gYW3VDZ7B1Gmvb9Yqo+eS2tGa/Ub2OZjYbOBx4hjG4HQesH4yBbZgvoWAZxo2Fc3FPcPcjgDOBT5vZiXEXNMzG0nb9PrAPcBiwFvjPaPyoXUczqwB+CXzO3Zt2NmuGcSN+HTOs35jYhvkSCvXAjLTn04E1MdUyZNx9TfR3A/Brwi7p+r5d7+jvhvgqHDKDrdOY2a7uvt7de909BfyQbc0Lo3IdzayQ8IV5p7v/Kho9ZrZjpvUbK9swX0LhWWA/M5tjZkXARcB9Mde0V8ys3Mwq+x4D7wFeIazXR6PZPgr8Np4Kh9Rg63Qf8I/R2SvHAo19zROjzYA29PcTtiWEdbzIzIrNbA6wH/D34a5vd5iZAf8NvObuN6RNGhPbcbD1GzPbMO4j3cM1EM5wWEw48v/luOsZgvWZSzij4UVgUd86AROAh4E3o7/j4651N9frLsKudzfhF9bHB1snwm75zdE2fRmYH3f9e7GOP4nW4SXCl8iUtPm/HK3jG8CZcdefxfq9g9A88hKwMBrOGivbcSfrNya2obq5EBGRfvnSfCQiIllQKIiISD+FgoiI9FMoiIhIP4WCiIj0UyiIiEg/hYJIFszssAFdIZ8zVF2wm9nnzKxsKJYlsrd0nYJIFszsUsJFVVfkYNkromVv2o3XJN29d6hrEdGegowpZjY7uvnJD6MboPzJzEoHmXcfM3sw6mX2STM7MBr/QTN7xcxeNLMnoq5RrgU+FN085UNmdqmZ3RTNf4eZfT+68coyMzsp6iXzNTO7I+39vm9mC6K6/nc07jPAVOBRM3s0GnexhZsnvWJm30p7fYuZXWtmzwDHmdk3zezVqFfO63PziUreifuSag0ahnIAZgM9wGHR83uASwaZ92Fgv+jxMcAj0eOXgWnR45ro76XATWmv7X8O3EG4R4cR+s5vAt5G+NH1XFotfd06JIHHgEOj5yuIbpZECIi3gDqgAHgEOC+a5sCFfcsidJlg6XVq0LC3g/YUZCxa7u4Lo8fPEYJiO1G3x8cDvzCzhcD/I9xRC+CvwB1m9gnCF3g2fufuTgiU9e7+sofeMhelvf+FZvY88AJwMOGOXAMdBTzm7hvdvQe4k3CnNoBeQs+cEIKnA7jVzD4AtGVZp8hOFcRdgEgOdKY97gUyNR8lgAZ3P2zgBHf/pJkdA5wNLDSzHebZyXumBrx/CiiIese8EjjK3bdGzUolGZaTqe/9Ph0eHUdw9x4zOxo4ldDr7xXAKVnUKbJT2lOQvOThpijLzeyD0H/z+LdHj/dx92fc/avAJkJf+M2E+/HuqSqgFWg0s0mEGyP1SV/2M8BJZlZrZkngYuDxgQuL9nSq3f1+4HOEG7uI7DXtKUg++wfg+2b2FaCQcFzgReA/zGw/wq/2h6NxbwFXRU1N39jdN3L3F83sBUJz0jJCE1WfW4AHzGytu7/LzK4GHo3e/353z3RPjErgt2ZWEs33+d2tSSQTnZIqIiL91HwkIiL91HwkY56Z3QycMGD0d9399jjqERnJ1HwkIiL91HwkIiL9FAoiItJPoSAiIv0UCiIi0u//A01nh9hikLmwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6570d04f98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators4_1.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第二步：调整树的参数：max_depth & min_child_weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一轮参数调整得到的n_estimators最优值（270），其余参数继续默认值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58967, std: 0.00360, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.59001, std: 0.00344, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.58984, std: 0.00335, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.59430, std: 0.00413, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.59398, std: 0.00483, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.59339, std: 0.00497, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.62953, std: 0.00748, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.61985, std: 0.00610, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.61495, std: 0.00603, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.68761, std: 0.00612, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.66533, std: 0.00592, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.64917, std: 0.00399, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 3, 'min_child_weight': 1},\n",
       " -0.5896663724918751)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=270,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([269.58421564, 249.85550466, 244.29024911, 358.84613028,\n",
       "        367.28623157, 364.88854117, 499.84778223, 497.83427429,\n",
       "        488.60857339, 620.78723249, 612.48472714, 609.9317625 ]),\n",
       " 'mean_score_time': array([1.13701944, 0.71854782, 0.67468791, 6.20780897, 1.43416986,\n",
       "        1.40976844, 6.01208839, 3.56870303, 3.34223723, 5.82850661,\n",
       "        5.33903632, 4.24978266]),\n",
       " 'mean_test_score': array([-0.58966637, -0.59000708, -0.58983998, -0.59430199, -0.59398388,\n",
       "        -0.59338662, -0.6295278 , -0.61984681, -0.61495283, -0.68761088,\n",
       "        -0.6653279 , -0.6491661 ]),\n",
       " 'mean_train_score': array([-0.52978153, -0.53225896, -0.53394335, -0.39778168, -0.41336159,\n",
       "        -0.4224088 , -0.21856275, -0.25690556, -0.28300739, -0.08992793,\n",
       "        -0.13572813, -0.16938852]),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 5, 5, 5, 7, 7, 7, 9, 9, 9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 5, 1, 3, 5, 1, 3, 5, 1, 3, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'rank_test_score': array([ 1,  3,  2,  6,  5,  4,  9,  8,  7, 12, 11, 10], dtype=int32),\n",
       " 'split0_test_score': array([-0.5832748 , -0.58416537, -0.58364215, -0.5868434 , -0.58630373,\n",
       "        -0.58529957, -0.61649725, -0.60902538, -0.60518143, -0.67881571,\n",
       "        -0.65933065, -0.6435725 ]),\n",
       " 'split0_train_score': array([-0.5315746 , -0.53406869, -0.53558717, -0.39889394, -0.41567429,\n",
       "        -0.42411055, -0.22046832, -0.25892767, -0.28352235, -0.09082997,\n",
       "        -0.13738486, -0.17077346]),\n",
       " 'split1_test_score': array([-0.58839362, -0.58830147, -0.58906884, -0.59377271, -0.59202288,\n",
       "        -0.59049372, -0.63092361, -0.61888252, -0.61449013, -0.68446509,\n",
       "        -0.6616912 , -0.64619893]),\n",
       " 'split1_train_score': array([-0.53009064, -0.53291947, -0.53457567, -0.39840636, -0.41454427,\n",
       "        -0.42428303, -0.21804989, -0.2581075 , -0.28283422, -0.09080103,\n",
       "        -0.13458631, -0.16831957]),\n",
       " 'split2_test_score': array([-0.59089588, -0.59113289, -0.59186093, -0.59486657, -0.59739997,\n",
       "        -0.59618851, -0.62994717, -0.62294628, -0.61275792, -0.68701104,\n",
       "        -0.66414771, -0.65108431]),\n",
       " 'split2_train_score': array([-0.52954851, -0.53214706, -0.53400059, -0.39794619, -0.4125474 ,\n",
       "        -0.42170224, -0.21913897, -0.25696304, -0.28464328, -0.08993568,\n",
       "        -0.13794682, -0.17146197]),\n",
       " 'split3_test_score': array([-0.59323538, -0.59351791, -0.59174472, -0.59713269, -0.59373705,\n",
       "        -0.59545235, -0.63040677, -0.62096126, -0.61979808, -0.69065169,\n",
       "        -0.66498383, -0.64987468]),\n",
       " 'split3_train_score': array([-0.52897367, -0.5315921 , -0.5332332 , -0.39739735, -0.4133994 ,\n",
       "        -0.42245451, -0.21762183, -0.25687126, -0.28336039, -0.09046587,\n",
       "        -0.13540914, -0.16974309]),\n",
       " 'split4_test_score': array([-0.59253304, -0.59291866, -0.59288417, -0.59889595, -0.60045775,\n",
       "        -0.59950079, -0.63986735, -0.6274209 , -0.62253887, -0.69711377,\n",
       "        -0.67648951, -0.65510186]),\n",
       " 'split4_train_score': array([-0.52872024, -0.53056748, -0.53232011, -0.39626456, -0.41064257,\n",
       "        -0.4194937 , -0.21753475, -0.25365831, -0.28067672, -0.08760711,\n",
       "        -0.13331352, -0.16664451]),\n",
       " 'std_fit_time': array([10.32301415,  2.61660943,  7.37832118,  3.7973713 ,  9.19735758,\n",
       "         5.53812901,  4.61200638,  4.35239758,  4.36256203,  6.75483885,\n",
       "         2.30605969,  2.50497997]),\n",
       " 'std_score_time': array([0.54951791, 0.12393962, 0.07974923, 8.69389441, 0.36686858,\n",
       "        0.32045345, 4.31076339, 0.87211063, 0.68011459, 0.04037225,\n",
       "        0.05839017, 0.81101662]),\n",
       " 'std_test_score': array([0.00360391, 0.00343821, 0.00334627, 0.00413148, 0.00482631,\n",
       "        0.0049652 , 0.00747774, 0.00610316, 0.00602545, 0.00611794,\n",
       "        0.00592044, 0.00398752]),\n",
       " 'std_train_score': array([0.00101429, 0.00118572, 0.00111754, 0.00090596, 0.00172074,\n",
       "        0.00175544, 0.00111071, 0.00179347, 0.00130598, 0.00120429,\n",
       "        0.00172645, 0.00173293])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.589666 using {'max_depth': 3, 'min_child_weight': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAELCAYAAAAoUKpTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPXV+PHPmezJJGxJgBARZFEUAmIEFIHQRxCLD0prcalW/Vm1WvXpokXt4lK1Wp8qtdI+7lvrbrHUIoKWRVFEsGyiCCJqAAmENSFkm/P7Y+5MZiaTzCRkMlnO+/XKi8m933vvmRuSM/f7vfd8RVUxxhhjGuOKdwDGGGPaPksWxhhjIrJkYYwxJiJLFsYYYyKyZGGMMSYiSxbGGGMismRhjDEmIksWxhhjIrJkYYwxJqLEeAfQUrKzs7Vfv37xDsMYY9qVVatW7VbVnEjtOkyy6NevHytXrox3GMYY066IyJfRtLNuKGOMMRFZsjDGGBORJQtjjDERdZgxC2NM01RXV1NcXMzhw4fjHYppBampqeTn55OUlNSs7S1ZGNNJFRcXk5mZSb9+/RCReIdjYkhVKS0tpbi4mP79+zdrH9YNZUwndfjwYXr06GGJohMQEXr06HFEV5GWLIzpxCxRdB5H+rPu9MlCVSl94kkq1n+MejzxDscYY9qkTj9mUb1tGyX33QeqJORk454wgcyiIjJOOQVXRka8wzPGmDah019ZJOfnM2jZu+Tdew/phYUcfHMBxddex2djTuGry3/Inmf/StXXX8c7TGM6nH379vHnP/+5WdvOmjWLQ4cOtXBELaOoqKjZ1SRee+01NmzY0KR9HT58mFGjRjF8+HBOOOEEbr311mYdO5JOnywAErt3p8vZZ5P/wAMMfm8ZfZ9+mm4XXUT1jh3svOsuPp80mc+nnsXO++6jfMUKtLo63iEb0+511GRxJEKTRTRSUlL497//zZo1a1i9ejXz589n+fLlLR5bp++GCiVJSWSMHkXG6FH0nPkLqr78krIlSyhbvIQ9zzzLnsefwJWVhfu003AXTSBj3DgSu3WLd9jGHJHb//kxG7YfaNF9Hp+Xxa3/fUKD62+66SY+//xzRowYwaRJk8jNzeWll16isrKS6dOnc/vtt1NeXs6MGTMoLi6mtraWX//61+zcuZPt27czceJEsrOzWbRoUdj9u91ufvzjH/PWW2/RrVs37r77bn7xi1/w1VdfMWvWLKZNm8bWrVu5+OKLKS8vB+Chhx7i1FNPZc6cOcyePZuFCxfyzTffMGHCBJYuXUqvXr3qHaeiooLLLruMDRs2MGTIECoqKvzrFixYwK233kplZSUDBgzgySefxO12069fP8477zx/7M899xwlJSXMnTuXJUuWcOedd/Lqq68C8PLLL3PNNdewb98+Hn/8ccaNGxd0fBHB7XYD3mdnqqurY3Ljgl1ZRJB89NF0/8EP6PvE4wx+/336PPhHMiedTvmKFWz/xUw2jT2NrRd+n90PP8LhjZ+hqvEO2Zh24Z577mHAgAGsXr2aSZMmsWnTJlasWMHq1atZtWoVS5cuZf78+eTl5bFmzRrWr1/PlClTuP7668nLy2PRokUNJgqA8vJyioqKWLVqFZmZmfzqV79i4cKFzJkzh9/85jcA5ObmsnDhQj766CNefPFFrr/+egCmT59Or169mD17NldccQW333572EQB8Je//IX09HTWrl3LL3/5S1atWgXA7t27ufPOO3nrrbf46KOPKCws5P777/dvl5WVxYoVK7j22mv5yU9+wqmnnsq0adO47777WL16NQMGDACgpqaGFStWMGvWLG6//XYAtm/fzre//W3/vmpraxkxYgS5ublMmjSJ0aNHH8FPJjy7smiCBHcGWZMnkzV5MurxcPjjjylbvISyxYvZ9cAD7HrgARLzevsHydNHj8aVmhrvsI2JqLErgNawYMECFixYwIknnghAWVkZmzZtYty4cdxwww3MnDmTs846q96n6sYkJyczZcoUAIYNG0ZKSgpJSUkMGzaMrVu3At5P4tdeey2rV68mISGBzz77zL/9n/70J4YOHcqYMWO44IILGjzO0qVL/UmmoKCAgoICAJYvX86GDRsYO3YsAFVVVZxyyin+7Xz7vOCCC/jpT3/a4P6/853vAHDSSSf5487Ly2PevHn+NgkJCaxevZp9+/Yxffp01q9fz9ChQ6M6T9GyZNFM4nKRNmwYacOGkXPdtVSXlFC+dCkHFy9m/z/msu/5F5DUVDLGjMFdVIS7aAJJDXwyMaazU1VuvvlmrrrqqnrrVq1axbx587j55puZPHmy/6ogkqSkJH93jMvlIiUlxf+6pqYGgAceeICePXuyZs0aPB4PqQEf7rZt24bL5WLnzp14PB5croY7YsJ1+6gqkyZN4vnnn4+4TWPdRr64ExIS/HE3pGvXrhQVFTF//vwWTxbWDdVCknJz6XruuRz10EMMXv4+Rz32GF3PPZfKzZv55rbb2Fw0kS3nTKdk1iwO/ec/aG1tvEM2Jq4yMzM5ePAgAGeccQZPPPEEZWVlgPcPdUlJCdu3byc9PZ2LLrqIG264gY8++qjetkdi//799O7dG5fLxbPPPkut83tZU1PDZZddxnPPPceQIUOCuo9CjR8/nr/97W8ArF+/nrVr1wIwZswYli1bxubNmwE4dOhQ0JXLiy++6P/Xd8XRnPe1a9cu9u3bB3jHT9566y2OO+64Ju0jGnZlEQOu5GTcp43FfdpY9Je3ULVlC2WLF1O2eAmljz5G6f89TEK3brjHj/cOkp92GgmZmfEO25hW1aNHD8aOHcvQoUM588wzufDCC/1/NN1uN3/961/ZvHkzN954Iy6Xi6SkJP7yl78AcOWVV3LmmWfSu3fvRsctIrnmmmv47ne/y8svv8zEiRPJcJ6tuvvuuxk3bhzjxo1jxIgRnHzyyUydOpUhQ4bU28fVV1/NZZddRkFBASNGjGDUqFEA5OTk8NRTT3HBBRdQWVkJwJ133sngwYMBqKysZPTo0Xg8Hv/Vx/nnn88VV1zBgw8+yCuvvNJg3Nu3b+eHP/wh8+bNY8eOHVxyySXU1tbi8XiYMWMGZ511VrPPSUOkowzIFhYWanuYKa92/37K3n2XsiVLKF+ylNr9+yExkfSTTsI9YQLuoiKS+1thNxN7n3zySdg/fib2fDN7Zmdnt+pxw/3MRWSVqhZG2tauLFpZQpcudJk6lS5Tp6K1tVSsWeMfJC/5/e8p+f3vSTq6b90geWEhkpwc77CNMZ2cJYs4koQE0keOJH3kSHJ/9lOqt2+nbMkSDi5ezL4XX2LvM8/iysgg49RTvYPkE8aT2MqfRIxp60aPHu3v5vF59tlnGTZsWIse580332TmzJlBy/r378+cOXOavC/fXU3tSUy7oURkCvBHIAF4TFXvCdNmBnAboMAaVb3QWf57YCreQfiFwP9oI8G2l26oaHkqKihfvtx/1VGzcycAqcOG4S6agHtCEanHD0EauUPDmMZYN1Tn0ya7oUQkAZgNTAKKgQ9FZK6qbghoMwi4GRirqntFJNdZfiowFihwmr4LTAAWxyretsaVlkbmxIlkTpyIqlK5caN/kHz3Q7PZ/aeHSMzJcRLHBCt8aIyJqVh2Q40CNqvqFgAReQE4GwgsfHIFMFtV9wKoaomzXIFUIBkQIAnYGcNY2zQRIfW440g97jiyf/QjavbsoWzpUsqWLOHAG/PZ9/IrSFIS6aNHewfJJxaRnJ8f77CNMR1ILJNFHyCwXGsxEPoM+mAAEVmGt6vqNlWdr6rvi8giYAfeZPGQqn4Sw1jblcTu3el6zjl0PecctLqaQ6s+cupXLWbnXXex8667SB44wD9InnbiiUiiDU8ZY5ovln9Bwt37GTrmkAgMAoqAfOAdERkKZANDnGUAC0VkvKouDTqAyJXAlQB9+/ZtucjbEUlKImPMaDLGjA4pfLi4fuHDiUVknHaaFT40xjRZLEdHi4GjAr7PB7aHafMPVa1W1S+AjXiTx3RguaqWqWoZ8AYwJvQAqvqIqhaqamFOTk5M3kR7U1f48Im6woenO4UPb/xFXeHDRx7l8GdW+NDET0ctUd7a81mA97mNYcOGMWLECAoLI45VN0ssk8WHwCAR6S8iycD5wNyQNq8BEwFEJBtvt9QW4CtggogkikgS3sFt64ZqIl/hw7y772LQ0iX0e/klsn/0I/TwYXbdfz9fTDubzf/1X3xzxx2ULVmC5wgmczemqTpqsjgSzZnPwmfRokWsXr262Ykqkph1Q6lqjYhcC7yJdzziCVX9WETuAFaq6lxn3WQR2QDUAjeqaqmIvAJ8C1iHt+tqvqr+M1axdgZBhQ+vv47qnSWULfXO07HvtX+w97nnvYUPTznFeZLcCh92Km/cBN+sa9l99hoGZ9a7W97P5rNomfksWktMRz1VdR4wL2TZbwJeK/Az5yuwTS1Qv/ykaTFJPXPp9r3v0e1738NTWcmhFR/6xzrKnP/AKUOG4J4wnsyiIlKHDUMSEuIctelI7rnnHtavX8/q1atZsGABr7zyCitWrEBVmTZtGkuXLmXXrl3k5eXxr3/9C/AW/uvSpQv3338/ixYtarRchm8+i3vvvZfp06f757PYsGEDl1xyCdOmTfPPZ5GamsqmTZu44IILWLlyJdOnT+fVV19l9uzZzJ8/P+r5LNauXcvIkSOB4PksMjIyuPfee7n//vv9VXN981k888wz/OQnP+H1119n2rRpnHXWWZx77rn+/fvms5g3bx633347b731VlBtKPDeMTl58mREhKuuuoorr7yyRX5GgewWGYMrJQX3uNNwjzvNW/jw88+9iWPR4rrCh9274x43zjtIPnasFT7saBq5AmgNNp/Fkc1nsWzZMvLy8igpKWHSpEkcd9xxjB8/PqrzFC1LFiaIiJAycCApAwfS4/LL6wofOk+S7//HP+oKHzrzdKT07x/vsE07Z/NZHNl8Fnl5eYB35r/p06ezYsWKFk8WVivCNMpX+LDPfb9n0LJ3Ofq5v9Hjssuo3bOHknvvZcuZ32bzGWew83e/o/z999GqqniHbNoJm8+iZeazKC8v929TXl7OggULWnziI7ArC9MEkphYV/jw5z+jets2Di5ZQtmSJex9/gX2PP2Mt/Dh2LHeq47x46zwoWmQzWfRMvNZ7Ny5k+nTpwPeJHfhhRf6u99aks1nYVqE59Ahypd/4B8k9xc+LCjAPWE87qIiUo8/3ubpaEOskGD82HwWptNypaeT+a2JZH7LKXz46af1Cx/m5voTR8Ypp+BKT4932MaYKFmyMC1OREgdMoTUIUPIvvpqakpLKXvnHcoWBxQ+TE4mfdQo/yC5FT40zWXzWbQO64Yyrcpf+HDxYsoWL6bK+aVJHjiAzKIi3BMmWOHDVmLdUJ2PdUOZdiOo8OFNM6nautU/O2Dp089Q+tjjuLp08RY+LJpghQ+NaSMsWZi4Su7Xj+79+tH9kkuoLSujfNl73quOpUs58K9/gctF2okn+id5Shk0yAbJjYkDSxamzUhwu8k6YzJZZ0xGPR4Or1/vfxhw1x/uZ9cf7icpL8+bOIqKSB89GpfzwJIxJrbsoTzTJonLRVpBATnXX0f/v7/KwCVL6PXbO0gZMoR9c17j6yuv4rMxp/D11dew98WXqN7ZaSdSbLc6atXZ1i5RvnHjRkaMGOH/ysrKYtasWc06fmPsysK0C2ELHzqD5EGFD4sm1BU+bKQ8g4k/X7K45pprmrztrFmzuOiii0jvYLdfv/baa5x11lkcf/zxUW9z7LHHsnr1agBqa2vp06eP/yG9lmS/Tabd8RU+7PXrXzHgrYUc8/o/yb3h57gy0il95FG2nnc+m04bx/abbubA/PnUtkBZCNPyAkuU33jjjdx3332cfPLJFBQUcOuttwLe8hVTp05l+PDhDB06lBdffJEHH3zQX6J84sSJDe7f7XYzc+ZMTjrpJE4//XRWrFhBUVERxxxzDHPneqfW2bp1K+PGjWPkyJGMHDmS9957D4A5c+Zw+umno6rs2LGDwYMH880334Q9TkVFBeeffz4FBQWcd9559UqUn3LKKYwcOZLvfe97/nIm/fr1Y+bMmYwaNYpRo0axefNm3nvvPebOncuNN97IiBEj+PzzzwFvifJRo0YxePBg3nnnnUbP6dtvv82AAQM4+uijo/wpRM+uLEy7FlT48Ic/pHbfPsreXUbZ4sUcXLSI/a+95i18WFjon6fDCh/Wd++Ke/l0z6ctus/juh/HzFEzG1xvJcpbrkS5zwsvvNBohdwjYcnCdCgJXbvS5aypdDlrKlpTQ8WaNf4nyUvuvZeSe+8l+eij6wbJTzoJSU6Od9idnpUoP7IS5b79z507l9/97ncRz01zWLIwHZY4pdTTTzqJ3J//nKribZQtWdxw4cMJ40ns0SPeYcdFY1cArcFKlB9ZiXKAN954g5EjR9KzZ88G2xwJG7MwnUZyfh+6f//79H3kEQYvf5/8P/+ZrKlTqVi9mh233MKm08bxxYzz2PXnP3N4wwY6SnWDtspKlLdMiXKf559/PmZdUGBXFqaTaqjw4cHFi9n9p4fY/eCfnMKHE7yzA44ZY4UPW5iVKG+ZEuXgTUQLFy7k4Ycfbva5iMRqQxkToqa0lLKl71C2eDHly5bhKSvzFj4cPdp5kryI5Pw+8Q7ziFltqPixEuXGdACJPXrQdfo5dJ1+DlpVxaGPPqJskXesY+dv72Tnb+8kZdBA5+6qItJGjLDCh6bDsysLY5qg8osvvBM8LVnCoQ9XQk1NQOHDItzjTiOha9d4hxmVjnJl0R5LlMfLkVxZWLIwpplqDx70Fj50kkftnj1BhQ8zi4pIHjiwzRY+7CjJwkTPuqGMiYOEzEyyppxB1pQzvIUP163zl1sPLnxYRPqY0aQVFJDUwINdxrR1liyMaQHicpE2fDhpw4eTc/31VO/c6cxHvoR9c+aw97nnAEjMzSVteAGpBQWkFQwnbegJuJw7cIxpyyxZGBMDST170m3GDLrNmIGnqorKTz6hYs1aKtaupWLdWg4ufMvb0OUiZeDAgARSQMrAgUhCQnzfgDEhLFkYE2Ou5GT/VYdPzd69HF63zp9ADi5YyL6XvffVS3o6aSecUJdAhg8nKUZP5RoTLXuC25g4SOzWDff48eRcdy19H32EQcvfZ8D8N8j7/b10/c538FRWUvr0M2y7/n/YPKGITROKKL7uOnY/+ijlH6zAU14e77dwxGw+i/qaM58FwB//+EeGDh3KCSecEJO5LMCuLIxpE0SE5H79SO7Xjy7TpgHU775a21j31XBSBg5oV91XNp9Ffc2Zz2L9+vU8+uijrFixwl88cerUqQwaNKhFY7MrC2PaKF/3VfcfXEyf/72PgQveZND773HUIw+TffXVJPbsycEFC/nm17/hi7PPZuPJo/jy4h9Q8r//y4EFC9r87IE2n0XLzGfxySefMGbMGNLT00lMTGTChAkxefbDriyMaUd83Vfu8eMBb2XT6i+/9F55rFlLxbp1lD79DFRXe9v37ElaQYH3CmRYQYN3X31z991UftKy81mkDDmOXrfc0uB6m8+iZeazGDp0KL/85S8pLS0lLS2NefPmUVgY8bGJJotpshCRKcAfgQTgMVW9J0ybGcBtgAJrVPVCZ3lf4DHgKGfdt1V1ayzjNaa9ib77aqF3g4DuK8/ZZ+M5fBhxSmDHk81n0fz5LIYMGcLMmTOZNGkSbreb4cOHkxiD8jMxSxYikgDMBiYBxcCHIjJXVTcEtBkE3AyMVdW9IpIbsItngLtUdaGIuAFPrGI1piNp8O6rtWupWLuOirVrObBgIbVFRVRu3gwuF93OvwBXehqutDQkPR1XUlKrxmzzWRzZfBaXX345l19+OQC33HIL+fn5De6vuWI5ZjEK2KyqW1S1CngBODukzRXAbFXdC6CqJQAicjyQqKoLneVlqto2b30wph1I7NYN94QJ/ruvBi9/n8TcXJLy80ns2hXUQ01pKVVff03lxo0c/nQjVV99Rc2uXdSWl6POPA8tyeazaLn5LEpKSgD46quv+Pvf/x6TeS1i2Q3VB/g64PtiYHRIm8EAIrIMb1fVbao631m+T0T+DvQH3gJuUtWW/x9rTCckIkhiojdROIUP1eNBDx/Gc6gCT8UhPBUV1B444N/GlZKKOFcfrvR0JCXliOpe2XwWLTefxXe/+11KS0tJSkpi9uzZdOvWrdnnpCExKyQoIt8DzlDVHzrfXwyMUtXrAtq8DlQDM4B84B1gKHA68DhwIvAV8CIwT1UfDznGlcCVAH379j3pyy+/jMl7MaYjiqaQoNbU4KmowHPImzy0osJ/lSEuF5LmJI84dV+1VzafRbBivIPTPvnA9jBtlqtqNfCFiGwEBjnL/6OqWwBE5DVgDN4E4qeqjwCPgLfqbCzehDGdmSQmkpCZSUJmJuDth9eqKm/icBJITWkpOB86JTEpeOwjNbVdPfthGhbLZPEhMEhE+gPbgPOBC0PavAZcADwlItl4u5+2APuAbiKSo6q7gG8BVn/cmDgTESQlBVdKSty6r0K1x/ksfHc1tScxSxaqWiMi1wJv4h2PeEJVPxaRO4CVqjrXWTdZRDYAtcCNqloKICI3AG+L93/VKuDRWMVqjGk+cbm8VxHp6UAPoH73lefAAWr37q1rnxaQPNLSjqj76oMPPmiJtxHRGWecwRlnnNEqx2qLYvqcharOA+aFLPtNwGsFfuZ8hW67ECiIZXzGdHaqGpPJmaLuvtq922lv3VexdqTj0/YEtzGdVGpqKqWlpfTo0SPms/k13n3lXH2Edl+lpgZfgbRw91VnoqqUlpYGPUfSVJYsjOmk8vPzKS4uZteuXfEOxU9F0Koq79e+fWhVlX/wHBEkKQlJTq77164+opaamnpED+tZsjCmk0pKSqJ///7xDqNRqkrV1q1Bc38c/vTTutpXvXqRNmxYXfXdE2zmwVixZGGMabNEhJT+/Unp37+u9lVlpbf2la94Ymjtq0GD6oonFhSQMqB9lW5vq2L2UF5rKyws1OZOOGKMad/8ta/8U9euw7N/PwCu9HRShw4NmvsjqWduhD12HtE+lGfJwhjT4fi7rwKKJ9brviooIK1gmDeBDB3q3Prb+bSFJ7iNMSYugrqvzvbWLw3bfbVggXcD676KyK4sjDGdVs2ePUGD552x+8q6oYwxpomCuq98d19t3Fi/+2p4AWkFBaSecEK7776ybihjjGki675qmF1ZGGNME9Xs2eO96nAGz+t1Xw0b5r3yKBjW5ruvrBvKGGNaSXvuvrJuKGOMaSVN7r5KSPB2XwU8fd7Wu6/sysIYY1pJXfeVk0DWrcPjFE90ZWR4777yjX8MK2iV7ivrhjLGmDauLXRfWTeUMca0cQ11Xx3esCHo+Y963VcBT5+3VveVXVkYY0wbF6n7yj1xIn3+975m7duuLIwxpoNI7N6dzKIiMouKAO/EUVVffulPHq1Rlj1ishCRAUCxqlaKSBHeqU6fUdV9sQ7OGGNMfeJy1eu+ijVXFG1eBWpFZCDwONAfeC6mURljjGlTokkWHlWtAaYDs1T1p0Dv2IZljDGmLYkmWVSLyAXAJcDrzrKk2IVkjDGmrYkmWVwGnALcpapfiEh/4K+xDcsYY0xbEnGAW1U3ANcDiEg3IFNV74l1YMYYY9qOiFcWIrJYRLJEpDuwBnhSRO6PfWjGGGPaimi6obqo6gHgO8CTqnoScHpswzLGGNOWRJMsEkWkNzCDugFuY4wxnUg0yeIO4E3gc1X9UESOATbFNixjjDFtSTQD3C8DLwd8vwX4biyDMsYY07ZEM8CdLyJzRKRERHaKyKsikt8awRljjGkboumGehKYC+QBfYB/OsuMMcZ0EtEkixxVfVJVa5yvp4CcGMdljDGmDYkmWewWkYtEJMH5uggojWbnIjJFRDaKyGYRuamBNjNEZIOIfCwiz4WsyxKRbSLyUDTHM8YYExvRzGfx/4CHgAcABd7DWwKkUSKSAMwGJgHFwIciMtd5ItzXZhBwMzBWVfeKSOiEs78FlkTzRowxxsROxCsLVf1KVaepao6q5qrqOXgf0ItkFLBZVbeoahXwAhBaeP0KYLaq7nWOVeJbISInAT2BBVG+F2OMMTESTTdUOD+Lok0f4OuA74udZYEGA4NFZJmILBeRKQAi4gL+ANzY2AFE5EoRWSkiK3ft2hV99MYYY5qkuclCmtkmdMLvRGAQUARcADwmIl2Ba4B5qvo1jVDVR1S1UFULc3JszN0YY2KluXNwh/7RD6cYOCrg+3xge5g2y1W1GvhCRDbiTR6nAONE5BrADSSLSJmqhh0kN8YYE1sNJgsROUj4pCBAWhT7/hAY5Mx/sQ04H7gwpM1reK8onhKRbLzdUltU9fsBcVwKFFqiMMaY+GkwWahq5pHsWFVrRORavHWlEoAnVPVjEbkDWKmqc511k0VkA1AL3KiqUd2Wa4wxpvWIajQ9Sm1fYWGhrly5Mt5hGGNMuyIiq1S1MFK75g5wG2OM6UQsWRhjjInIkoUxxpiIIt4628BdUfuBlcDPnfktjDHGdGDRPGdxP97nI57De9vs+UAvYCPwBN4H6owxxnRg0XRDTVHVh1X1oKoeUNVHgG+r6otAtxjHZ4wxpg2IJll4nDLiLudrRsC6jnHfrTHGmEZFkyy+D1wMlDhfFwMXiUgacG0MYzPGGNNGRByzcAaw/7uB1e+2bDjGGGPaoohXFiKSLyJzRKRERHaKyKsikt8awRljjGkboumGehKYC+ThnY/in84yY4wxnUQ0ySJHVZ9U1Rrn6ynAJo8wxphOJJpksVtELhKRBOfrIsAqwxpjTCcSTbL4f8AM4BtgB3AucFksgzLGGNO2REwWqvqVqk5T1RxVzVXVc4DvtEJsrUJVWbtrLbsrdtNRyrUbY0xLa+60qj8DZrVkIPGy5/Aevj/POzFfakIqee48+rj70Mfdh/zMfP/rPpl9yErOinO0xhgTH81NFtKiUcRRelI6D33rIYrLitlWto1tB7exrWwb/yn5D2XVZUFtM5Mz65JHSELJc+eRlhjNbLPGGNP+NDejS79kAAAUwElEQVRZdJj+mrTENCYcNSHsuv2V+70JxEkivoTyxf4veHfbu1TWVga1757anXx3vv9KxJ9Q3Pn0cvciyZXUGm/JGGNaXIPJooHS5OC9qugUH6G7pHShS0oXju9xfL11qkrp4VKKDxbXJRQnqazbvY6FXy6kRmv87V3iIjc9NyiBBCaUnLQcElwJrfn2jDEmag0mC1XNbM1A2hsRITstm+y0bEbkjqi3vsZTQ8mhEraVbQtKKNvLtrN8x3J2HdqFBuTiRFcieRl5QVclgVcp3VK6IdJhev+MMe1Mc7uhTASJrkTy3HnkufM4udfJ9dZX1VaxvWy7P4kUlxWz7aA3mbz95dvsrdwb1D4tMa3eeEmfTG9CyXPnkZlsud0YEzuWLOIkOSGZfl360a9Lv7Dry6vLgwbcA79W7lxJeXV5UPus5Kz6d3A5X3nuPFITU1vhXRljOipLFm1URlIGg7sNZnC3wfXWqap/8D3oLq7ybWzau4klXy+hylMVtE12Wna9u7h8twn3yrDBd2NM4yxZtEMiQtfUrnRN7coJ2SfUW+9RD7srdgeNl/i6vNbsWsObW9+kVmv97RMkgZ7pPYMG3AO/ctJzcEk0D/sbYzoqSxYdkO/Oq9z0XE7MPbHe+mpPNTvLd9aNlxwsZnv5drYd3MaybcvYVbErqH2yKznoYcU+md6uLd8AfNeUrjb4bkwHZ8miE0pyJZGfmU9+ZvhpSQ7XHPYnD99Via+7a33pevZX7g9qn56YXu8OLl9yyc/MJyMpozXeljEmhixZmHpSE1M5pssxHNPlmLDry6rKgu/gchJL8cFiPtjxARU1FUHtu6Z0Dboq6ZNRd3twnjuPlISU1nhbxpgjYMnCNJk72c2x3Y/l2O7H1lunquyt3Bv2Lq6Nezey6OtFVHuqg7bJTcsNSh6Bz5f0TO9Josv+mxoTb/ZbaFqUiNA9tTvdU7szLGdYvfUe9fgfVgx86n1b2TZW7VzFvC/m4VGPv32CJNAro5f/eZLA50v6uPuQnZZt4yXGtAJLFqZVucRFr4xe9MroxUk9T6q3vrq2mm/Kv/GPkQSOlywtXkrp4eB5t1ISUoIH30MeWMxKzrJkYkwLsGRh2pSkhCSOyjqKo7KOCru+oqbCfxtw8cHioKfg1+xaw8Gqg0Ht3UnuegPugUklPSm9Nd6WMe1eTJOFiEwB/ggkAI+p6j1h2swAbsNbtHCNql4oIiOAvwBZQC1wl6q+GMtYTfuQlpjGgK4DGNB1QNj1B6oOhB0v+erAV7y//X0O1x4Oat89tXtQMgks8tg7ozfJCcmt8baMafMkVrPDiUgC8BkwCSgGPgQuUNUNAW0GAS8B31LVvSKSq6olIjIYUFXdJCJ5wCpgiKrua+h4hYWFunLlypi8F9Mx+CoF+8ZJtpdvDyryuKNsR1ClYEGCKgWHlp3PTc+1SsGm3RORVapaGKldLK8sRgGbVXWLE9ALwNnAhoA2VwCzVXUvgKqWOP9+5mugqttFpATIARpMFsZEElgpeHjO8Hrraz21lBwqobgsuHur+GAxK75ZQcmWknqVgntn9A6+gysgqfRI7WHjJabDiGWy6AN8HfB9MTA6pM1gABFZhrer6jZVnR/YQERGAcnA57EL1RhIcCXQ292b3u7eYddX1Vaxo3yHvw5XYHfXoq8XsefwnqD2gdP0Zqdlk5mcSVZyFlkpWXWvfV/OMnvmxLRVsUwW4T5ShfZ5JQKDgCIgH3hHRIb6uptEpDfwLHCJasD9lL4DiFwJXAnQt2/flovcmDCSE5I5Outojs46Ouz6Q9WH6t3B5UsoG/ds5GD1wXoPLIZKSUghK7kumWQmZ5KVkhW0LDTB+L7PSMqwKxkTM7FMFsVA4C0t+cD2MG2Wq2o18IWIbMSbPD4UkSzgX8CvVHV5uAOo6iPAI+Ads2jh+I1pkvSkdAZ1G8SgboMabFNVW8WBqgMcrDrIgaoDHKise+1f5ntdeYDdFbvZsn8LB6oOUFZVFtQNFsolruAkE5hgUuqSTGCC8SWjzORMqzxsGhXLZPEhMEhE+gPbgPOBC0PavAZcADwlItl4u6W2iEgyMAd4RlVfjmGMxrSq5IRk/7hJU3nUQ1l1WfgEU3kgONE4r3ce2ulfF/rkfKi0xLTICSYli8yk+lc7aYlpdlXTwcUsWahqjYhcC7yJdzziCVX9WETuAFaq6lxn3WQR2YD3FtkbVbVURC4CxgM9RORSZ5eXqurqWMVrTFvnEpf/j3dzHK45HDnBBCSiHWU72Fi1kQNVB+pNthUq0ZUY8coldFlWkjf5uJPcdldZOxCzW2dbm906a0zs1HhqKKsq8yeS/VX7G+9KqzzAweqD/oQUOH9KOO4kd71xmLAJJjmr3jK7KeDItIVbZ40xHUSiK9E/4VZTqSoVNRX+qxhfAgmbYJzXXx740r8umpsCwo3RBHaXhUs6mcmZZCRl2MReUbJkYYyJKREhPSmd9KR0emX0avL21bXVDd4AELr8QJX3poAv9n/hXxfppoBwVzVBd5yFGaPpjDcFWLIwxrRpSQlJ9EjrQY+0Hk3e1qMeyqvLG08wIct2HdrlXx46l32otMS0iLc0N/R8TXu7KcCShTGmw/LdTpyZnNms7StrK4O6xyJ1pe0o38Fnez/z3upcXdbovhMlscHEEun5mnjcFGDJwhhjGpCSkEJOeg456TlN3rbWU+u/1flAdQMJJmTZtrJt/mQUWKcsHF/3WWZyJkOzh3Lbqbc1811Gx5KFMcbEQIIrgS4pXeiS0qXJ24beFOBLLIF3mAVe7aQmpsbgHQSzZGGMMW3Mkd4UEAt2z5gxxpiILFkYY4yJyJKFMcaYiCxZGGOMiciShTHGmIgsWRhjjInIkoUxxpiILFkYY4yJyJKFMcaYiCxZGGOMiciShTHGmIgsWRhjjInIkoUxxpiILFkYY4yJyJKFMcaYiCxZGGOMiciShTHGmIgsWRhjjInIkoUxxpiILFkYY4yJyJKFMcaYiCxZGGOMiciShTHGmIgsWRhjjIkopslCRKaIyEYR2SwiNzXQZoaIbBCRj0XkuYDll4jIJufrkljGaYwxpnGJsdqxiCQAs4FJQDHwoYjMVdUNAW0GATcDY1V1r4jkOsu7A7cChYACq5xt98YqXmOMMQ2L5ZXFKGCzqm5R1SrgBeDskDZXALN9SUBVS5zlZwALVXWPs24hMCWGsRpjjGlELJNFH+DrgO+LnWWBBgODRWSZiCwXkSlN2NYYY0wriVk3FCBhlmmY4w8CioB84B0RGRrltojIlcCVAH379j2SWI0xxjQillcWxcBRAd/nA9vDtPmHqlar6hfARrzJI5ptUdVHVLVQVQtzcnJaNHhjjDF1YpksPgQGiUh/EUkGzgfmhrR5DZgIICLZeLultgBvApNFpJuIdAMmO8uMMcbEQcy6oVS1RkSuxftHPgF4QlU/FpE7gJWqOpe6pLABqAVuVNVSABH5Ld6EA3CHqu6JVazGGGMaJ6r1hgLapcLCQl25cmW8wzDGmHZFRFapamGkdvYEtzHGmIgsWRhjjInIkoUxxpiILFkYY4yJyJKFMcaYiGL5BHf74KmF8l2AgEiYfwn+XlyNtG3gXwn3QLoxxrQfliwO7YE/HNsKB2pKgnEFvI52W1eYZZG2DbdNI9s2KVFyhMk1HucrzIeDqM9XQ8d31Z2Hel8NLW9Km3DrE1pgH81tYx+MOipLFilumHo/oOB75kQ14Ptw/3oaWUczttH6x1dPFG1D/m1KW/82nvrHbtZ7D9jWExp7NO899LxFe76cbVv7fJkGNJBUXAktkLRaOrmGJtV4xBDuPDVjHylZkDcipj9ZSxZJaXDy5fGOwrRHUSUcT8hrT13SafSrGW08tUe+j2a1aWx9bSscI2C9J5o2oeepBY4f7w8QfQrhirdjeghLFsY0l3W7GB/VKBJOY0mnNrqk1FCb5IyYv0VLFsYYc6T8Hxxc8Y4kZjruOzPGGNNiLFkYY4yJyJKFMcaYiGzMohNRVVTBo+q921RBUf+drx5nvfraOm0I2qb+tkGvA44Tdrlv2wb243GCCbetx7m9tm5Z3X48zsLQffripl68de83dJ+B7zU0Fk/AMQmJMfD8EfJefcQZEPc/0hHwjfjb+NbVtW1oHUHrGtiPBO8rkIg0elzf/uq2C7eu4eP69xf4fkO2aey4de8x/DoRCbOsfqyNnaOGz01dXGHfU7191z9u3XuM/DOud9wmnKOkBBfdMpKJpU6fLPYfquaqv64M+CWv/4cm3B+A0D+sUPdHpf4flLo/doF/sCD4j13gHyzC/REK2WfdNlov/sAYAv5WGWM6oBFHdeW1H4+N6TE6fbJAnGfInE8I4gLBhQi4Aj+1OJ/A/MvxZXfxf7oI3Mb3acr3qcIV8Nr/kK+zrSvgdeg+6/YX/Kk0MIbQfbqk7hOT/5OP0yZ4f3X7rLc84NNUaAz1X4ffVoJibGA5jew7cFv/92G2bWA/rno/n+CfIWH20/DPVoJ+Tv5tw5zX4J9t3Sd3X872f7gg4EMDGtSorm3dOq23Tut9EAjdn/9DRMhx69o2tO+6/dR9sKm/78aOW/cewx3Xad3QcaM8R40d17efgMPVaxuuff14Gj5HjR2XgHZhY2zkZxMYa2M/G9/32e7YXlWAJQu6pCXx0o9OiXcYxhjTptkAtzHGmIgsWRhjjInIkoUxxpiILFkYY4yJyJKFMcaYiCxZGGOMiciShTHGmIgsWRhjjIlIAp80bM9EZBfw5RHsIhvY3ULhtCSLq2ksrqaxuJqmI8Z1tKrmRGrUYZLFkRKRlapaGO84QllcTWNxNY3F1TSdOS7rhjLGGBORJQtjjDERWbKo80i8A2iAxdU0FlfTWFxN02njsjELY4wxEdmVhTHGmIg6VbIQkSdEpERE1jewXkTkQRHZLCJrRWRkG4mrSET2i8hq5+s3rRTXUSKySEQ+EZGPReR/wrRp9XMWZVytfs5EJFVEVojIGieu28O0SRGRF53z9YGI9GsjcV0qIrsCztcPYx1XwLETROQ/IvJ6mHWtfr6iiCme52qriKxzjrsyzPrY/T56Z9vqHF/AeGAksL6B9d8G3sA74dkY4IM2ElcR8HoczldvYKTzOhP4DDg+3ucsyrha/Zw558DtvE4CPgDGhLS5Bvg/5/X5wIttJK5LgYda+/+Yc+yfAc+F+3nF43xFEVM8z9VWILuR9TH7fexUVxaquhTY00iTs4Fn1Gs50FVEereBuOJCVXeo6kfO64PAJ0CfkGatfs6ijKvVOeegzPk2yfkKHRQ8G3jaef0K8F/im9s2vnHFhYjkA1OBxxpo0urnK4qY2rKY/T52qmQRhT7A1wHfF9MG/gg5TnG6Ed4QkRNa++DO5f+JeD+VBorrOWskLojDOXO6L1YDJcBCVW3wfKlqDbAf6NEG4gL4rtN18YqIHBXrmByzgF8AngbWx+N8RYoJ4nOuwJvkF4jIKhG5Msz6mP0+WrIIFu4TS1v4BPYR3kfyhwN/Al5rzYOLiBt4FfiJqh4IXR1mk1Y5ZxHiiss5U9VaVR0B5AOjRGRoSJO4nK8o4von0E9VC4C3qPs0HzMichZQoqqrGmsWZlnMzleUMbX6uQowVlVHAmcCPxaR8SHrY3a+LFkEKwYCPyXkA9vjFIufqh7wdSOo6jwgSUSyW+PYIpKE9w/y31T172GaxOWcRYornufMOeY+YDEwJWSV/3yJSCLQhVbsgmwoLlUtVdVK59tHgZNaIZyxwDQR2Qq8AHxLRP4a0qa1z1fEmOJ0rnzH3u78WwLMAUaFNInZ76Mli2BzgR84dxSMAfar6o54ByUivXz9tCIyCu/PrbQVjivA48Anqnp/A81a/ZxFE1c8zpmI5IhIV+d1GnA68GlIs7nAJc7rc4F/qzMyGc+4Qvq1p+EdB4opVb1ZVfNVtR/ewet/q+pFIc1a9XxFE1M8zpVz3AwRyfS9BiYDoXdQxuz3MbEldtJeiMjzeO+SyRaRYuBWvIN9qOr/AfPw3k2wGTgEXNZG4joXuFpEaoAK4PxY/4FxjAUuBtY5/d0AtwB9A2KLxzmLJq54nLPewNMikoA3Ob2kqq+LyB3ASlWdizfJPSsim/F+Qj4/xjFFG9f1IjINqHHiurQV4gqrDZyvSDHF61z1BOY4n4ESgedUdb6I/Ahi//toT3AbY4yJyLqhjDHGRGTJwhhjTESWLIwxxkRkycIYY0xEliyMMcZEZMnCGGNMRJYsjGllTpnpZj1N7pTHzmuJfRnTFJYsjGlfLgXyIjUypqVZsjCdloj0E5FPReQxEVkvIn8TkdNFZJmIbBKRUc7Xe+KdCOc9ETnW2fZnIvKE83qYs316A8fpISILnH08TECxNxG5SLwTE60WkYedp6wRkTIR+YOIfCQibzslO84FCoG/Oe3TnN1c57RbJyLHxfKcmc7LkoXp7AYCfwQKgOOAC4HTgBvwlhD5FBivqicCvwHudrabBQwUkenAk8BVqnqogWPcCrzr7GMuTlkSERkCnIe3kugIoBb4vrNNBvCRU2F0CXCrqr4CrAS+r6ojVLXCabvbafcXJ25jWlynqg1lTBhfqOo6ABH5GHhbVVVE1gH98FY5fVpEBuEt9eyr2eURkUuBtcDDqrqskWOMB77jbPcvEdnrLP8vvBVLP3Tq/aThnW8CvHMpvOi8/isQruKvj2/dKt9xjGlplixMZ1cZ8NoT8L0H7+/Hb4FFqjpdvBMtLQ5oPwgoI7oxhHBF2AR4WlVvbub2Pr6Ya7HfaRMj1g1lTOO6ANuc15f6FopIF7zdV+OBHs54QkOW4nQviciZQDdn+dvAuSKS66zrLiJHO+tceCvngrdr7F3n9UG8844b06osWRjTuN8DvxORZUBCwPIHgD+r6mfA5cA9vj/6YdwOjBeRj/DOQfAVgKpuAH6Fd5rMtcBCvOXEAcqBE0RkFfAt4A5n+VPA/4UMcBsTc1ai3Jg2SETKVNUd7ziM8bErC2OMMRHZlYUxLURELgP+J2TxMlX9cTziMaYlWbIwxhgTkXVDGWOMiciShTHGmIgsWRhjjInIkoUxxpiILFkYY4yJ6P8DPeDcTHRlPdAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f76601160b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "进行精细调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [2, 3, 4], 'min_child_weight': [1, 2, 3]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = [2,3,4]\n",
    "min_child_weight = [1,2,3]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.61181, std: 0.00335, params: {'max_depth': 2, 'min_child_weight': 1},\n",
       "  mean: -0.61155, std: 0.00333, params: {'max_depth': 2, 'min_child_weight': 2},\n",
       "  mean: -0.61172, std: 0.00355, params: {'max_depth': 2, 'min_child_weight': 3},\n",
       "  mean: -0.59872, std: 0.00306, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.59837, std: 0.00308, params: {'max_depth': 3, 'min_child_weight': 2},\n",
       "  mean: -0.59878, std: 0.00317, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.59142, std: 0.00395, params: {'max_depth': 4, 'min_child_weight': 1},\n",
       "  mean: -0.59205, std: 0.00368, params: {'max_depth': 4, 'min_child_weight': 2},\n",
       "  mean: -0.59197, std: 0.00391, params: {'max_depth': 4, 'min_child_weight': 3}],\n",
       " {'max_depth': 4, 'min_child_weight': 1},\n",
       " -0.5914234985131468)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=270,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 81.3984036 ,  78.59876037,  69.09772577,  85.97326927,\n",
       "        105.92188163,  93.90082378, 122.87801332, 119.20342064,\n",
       "        124.04248023]),\n",
       " 'mean_score_time': array([1.95673571, 0.17163854, 0.32839451, 0.26771274, 0.25782495,\n",
       "        1.14840221, 0.31463232, 0.34226193, 0.39129205]),\n",
       " 'mean_test_score': array([-0.61181471, -0.6115482 , -0.6117201 , -0.59871847, -0.59836988,\n",
       "        -0.59877699, -0.5914235 , -0.5920535 , -0.59197478]),\n",
       " 'mean_train_score': array([-0.60046506, -0.60047882, -0.6006433 , -0.57194389, -0.57229535,\n",
       "        -0.5726189 , -0.53949433, -0.54088527, -0.5421202 ]),\n",
       " 'param_max_depth': masked_array(data=[2, 2, 2, 3, 3, 3, 4, 4, 4],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 2, 3, 1, 2, 3, 1, 2, 3],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 2, 'min_child_weight': 1},\n",
       "  {'max_depth': 2, 'min_child_weight': 2},\n",
       "  {'max_depth': 2, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 2},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 4, 'min_child_weight': 1},\n",
       "  {'max_depth': 4, 'min_child_weight': 2},\n",
       "  {'max_depth': 4, 'min_child_weight': 3}],\n",
       " 'rank_test_score': array([9, 7, 8, 5, 4, 6, 1, 3, 2], dtype=int32),\n",
       " 'split0_test_score': array([-0.60596454, -0.60590419, -0.60605509, -0.59322149, -0.59294242,\n",
       "        -0.59356787, -0.58495971, -0.5861683 , -0.58521185]),\n",
       " 'split0_train_score': array([-0.60222784, -0.60233808, -0.60256751, -0.5732688 , -0.57382774,\n",
       "        -0.57425954, -0.53965962, -0.54119003, -0.54282575]),\n",
       " 'split1_test_score': array([-0.61247824, -0.61176961, -0.61192972, -0.59803871, -0.59773322,\n",
       "        -0.5974313 , -0.59052951, -0.59131753, -0.59135122]),\n",
       " 'split1_train_score': array([-0.60110109, -0.60079776, -0.60078382, -0.57223694, -0.57244486,\n",
       "        -0.57277531, -0.54014825, -0.54115781, -0.54247427]),\n",
       " 'split2_test_score': array([-0.61118152, -0.61069301, -0.61056292, -0.59954785, -0.59871312,\n",
       "        -0.599159  , -0.59050858, -0.5909291 , -0.59171867]),\n",
       " 'split2_train_score': array([-0.60040636, -0.60037915, -0.60052426, -0.57228741, -0.57221748,\n",
       "        -0.57270223, -0.53905037, -0.54042952, -0.54115113]),\n",
       " 'split3_test_score': array([-0.61612889, -0.61591236, -0.61699674, -0.60219069, -0.60201193,\n",
       "        -0.60285776, -0.59505348, -0.5951063 , -0.59502224]),\n",
       " 'split3_train_score': array([-0.59905134, -0.59915028, -0.59948255, -0.5706224 , -0.57140215,\n",
       "        -0.57141443, -0.5397576 , -0.54098791, -0.54155271]),\n",
       " 'split4_test_score': array([-0.61332081, -0.6134624 , -0.61305645, -0.60059419, -0.60044935,\n",
       "        -0.60086965, -0.59606763, -0.5967477 , -0.59657134]),\n",
       " 'split4_train_score': array([-0.59953865, -0.59972882, -0.59985834, -0.5713039 , -0.5715845 ,\n",
       "        -0.57194298, -0.53885582, -0.54066106, -0.54259712]),\n",
       " 'std_fit_time': array([ 4.08713137, 10.89146012,  5.60920792,  5.29802851, 11.1008788 ,\n",
       "         1.59431283,  1.73240891,  4.89491001,  3.39394844]),\n",
       " 'std_score_time': array([1.03558485, 0.00462065, 0.17681786, 0.08796472, 0.05925015,\n",
       "        1.16335538, 0.02282106, 0.01657559, 0.10982533]),\n",
       " 'std_test_score': array([0.00334511, 0.00332709, 0.00355267, 0.00306409, 0.0030845 ,\n",
       "        0.00316624, 0.00395373, 0.00368223, 0.00391421]),\n",
       " 'std_train_score': array([0.00112891, 0.00108591, 0.00106771, 0.00090721, 0.00085788,\n",
       "        0.00096237, 0.00047519, 0.00029519, 0.00064989])}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.cv_results_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "再次调整若学习器分类数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit_1(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=100):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgb_param['num_class'] = 3\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print (cvresult)\n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('my_preds4_2_3_699.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-mlogloss-mean']\n",
    "        test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-mlogloss-mean']\n",
    "        train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( 'n_estimators4_2_3_699.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\" )\n",
    "    print(logloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     test-mlogloss-mean  test-mlogloss-std  train-mlogloss-mean  \\\n",
      "0              1.041913           0.000569             1.041587   \n",
      "1              0.993855           0.000612             0.993250   \n",
      "2              0.952544           0.000789             0.951504   \n",
      "3              0.916851           0.000686             0.915522   \n",
      "4              0.886035           0.000506             0.884479   \n",
      "5              0.859159           0.000504             0.857269   \n",
      "6              0.835443           0.000587             0.833216   \n",
      "7              0.814602           0.000536             0.812136   \n",
      "8              0.796386           0.000551             0.793531   \n",
      "9              0.780327           0.000466             0.777230   \n",
      "10             0.766551           0.000466             0.763105   \n",
      "11             0.754260           0.000557             0.750385   \n",
      "12             0.743132           0.000473             0.738944   \n",
      "13             0.733218           0.000496             0.728709   \n",
      "14             0.724356           0.000552             0.719519   \n",
      "15             0.716544           0.000516             0.711296   \n",
      "16             0.709451           0.000689             0.703955   \n",
      "17             0.702972           0.000721             0.697137   \n",
      "18             0.697120           0.000966             0.691027   \n",
      "19             0.692097           0.000831             0.685811   \n",
      "20             0.687193           0.000937             0.680616   \n",
      "21             0.682662           0.000915             0.675816   \n",
      "22             0.678875           0.001170             0.671750   \n",
      "23             0.675212           0.001182             0.667853   \n",
      "24             0.671866           0.001200             0.664308   \n",
      "25             0.668859           0.001185             0.661038   \n",
      "26             0.666264           0.001266             0.658172   \n",
      "27             0.663646           0.001301             0.655290   \n",
      "28             0.661519           0.001499             0.652833   \n",
      "29             0.659230           0.001715             0.650302   \n",
      "..                  ...                ...                  ...   \n",
      "240            0.593624           0.002615             0.538375   \n",
      "241            0.593564           0.002538             0.538161   \n",
      "242            0.593590           0.002529             0.537957   \n",
      "243            0.593609           0.002476             0.537754   \n",
      "244            0.593605           0.002449             0.537575   \n",
      "245            0.593514           0.002416             0.537326   \n",
      "246            0.593556           0.002476             0.537116   \n",
      "247            0.593592           0.002457             0.536902   \n",
      "248            0.593592           0.002485             0.536686   \n",
      "249            0.593597           0.002439             0.536446   \n",
      "250            0.593589           0.002412             0.536201   \n",
      "251            0.593437           0.002465             0.535980   \n",
      "252            0.593419           0.002513             0.535732   \n",
      "253            0.593387           0.002542             0.535524   \n",
      "254            0.593359           0.002566             0.535301   \n",
      "255            0.593371           0.002539             0.535078   \n",
      "256            0.593325           0.002548             0.534884   \n",
      "257            0.593319           0.002523             0.534621   \n",
      "258            0.593298           0.002482             0.534446   \n",
      "259            0.593244           0.002489             0.534214   \n",
      "260            0.593235           0.002443             0.533972   \n",
      "261            0.593229           0.002446             0.533738   \n",
      "262            0.593215           0.002470             0.533513   \n",
      "263            0.593166           0.002389             0.533305   \n",
      "264            0.593156           0.002402             0.533039   \n",
      "265            0.593112           0.002452             0.532805   \n",
      "266            0.593056           0.002531             0.532579   \n",
      "267            0.593076           0.002601             0.532368   \n",
      "268            0.593058           0.002606             0.532150   \n",
      "269            0.592941           0.002567             0.531967   \n",
      "\n",
      "     train-mlogloss-std  \n",
      "0              0.000529  \n",
      "1              0.000529  \n",
      "2              0.000710  \n",
      "3              0.000885  \n",
      "4              0.000925  \n",
      "5              0.000947  \n",
      "6              0.000997  \n",
      "7              0.000959  \n",
      "8              0.001114  \n",
      "9              0.001263  \n",
      "10             0.001348  \n",
      "11             0.001321  \n",
      "12             0.001568  \n",
      "13             0.001597  \n",
      "14             0.001684  \n",
      "15             0.001737  \n",
      "16             0.001633  \n",
      "17             0.001684  \n",
      "18             0.001708  \n",
      "19             0.001946  \n",
      "20             0.001934  \n",
      "21             0.001913  \n",
      "22             0.001739  \n",
      "23             0.001771  \n",
      "24             0.001798  \n",
      "25             0.001948  \n",
      "26             0.001896  \n",
      "27             0.001907  \n",
      "28             0.001715  \n",
      "29             0.001720  \n",
      "..                  ...  \n",
      "240            0.001501  \n",
      "241            0.001537  \n",
      "242            0.001572  \n",
      "243            0.001587  \n",
      "244            0.001599  \n",
      "245            0.001600  \n",
      "246            0.001615  \n",
      "247            0.001611  \n",
      "248            0.001601  \n",
      "249            0.001624  \n",
      "250            0.001567  \n",
      "251            0.001535  \n",
      "252            0.001580  \n",
      "253            0.001538  \n",
      "254            0.001555  \n",
      "255            0.001544  \n",
      "256            0.001558  \n",
      "257            0.001532  \n",
      "258            0.001518  \n",
      "259            0.001531  \n",
      "260            0.001581  \n",
      "261            0.001585  \n",
      "262            0.001589  \n",
      "263            0.001593  \n",
      "264            0.001541  \n",
      "265            0.001516  \n",
      "266            0.001522  \n",
      "267            0.001519  \n",
      "268            0.001509  \n",
      "269            0.001552  \n",
      "\n",
      "[270 rows x 4 columns]\n",
      "logloss of train :\n",
      "0.5461167965862244\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcHHWd//HXp3vuycxkJpkkJCEQwg0iR7hEBWXlUrlUBIUVLxYVEXfVxf35UH8o67EurgfqDxFYUYioCAioKIciCCaEGwzGkJCEHDNJ5r67P78/vtWTzqQn00mmp2am38/HozLdVdXVn+qa9HvqW1XfMndHREQEIBF3ASIiMn4oFEREZJBCQUREBikURERkkEJBREQGKRRERGSQQkEki5n9h5ldH3cdInFRKEwwZjbFzFaa2XuyxtWY2Stm9s6scQvN7G4z22JmLWb2gpldbWb10fSLzSxlZh3RsMLMPlLg2k8yszWFfI+dkased/9Pd/9Qgd5vpZn9UyGWXQhjtb0m2ucy2SkUJhh37wAuAb5lZo3R6K8DS9z9FwBm9jrgIeAR4EB3nwqcBgwAr81a3F/cfYq7TwHeCXzdzI4YmzWRnWFmJXHXIEXC3TVMwAG4CbgVOAnYBOyRNe3PwHdGeP3FwJ+HjPsr8J6s52cCzwMthJA5KGvaQdG4lmieM7OmnQG8ALQDa4FPAdVAN5AGOqJh9jDrdS1wT/T6x4EFeXweBwK/BzYDy4DzdqUe4IvAT6LX7Q048H5gNbAFuBQ4GngmWvfvZr3PAuCBaHs0Az8FpkbTbo7eqzt6r8/k8RmvBP49eq9eoCR6vjZal2XAyTk+i+OA9UAya9w5wDPR42OAJUAbsAG4ZpjP9CRgzTDT6oAfA03AKuBzQCKalgT+O/oMXgYuiz7HkmGWtRL4p2GmfRhYHm3XuzK/M4AB3wQ2Aq3RZ3TocNs77v+vE2mIvQANu7jhoB5YF/3He3/W+GogBZw0wusvJisUoi+6FmD/6Pn+QCfwFqAU+Ez0n7Mser4c+I/o+Zuj/4AHRK9dB7whq84jo8fDfslk1XFT9AVwTPQl+FNg0QivqSZ8ab8/es2R0edyyM7WQ+5Q+AFQAZwC9AB3ADOAOdGX0onR/PtGn1c50Aj8CfifrGVv8+W3o884a/6ngD2BSuCAaD1nZ9WXMzCBfwBvyXr+c+DK6PFfgIuix1OA44ZZxrDbixAIdwI1UR0vAR+Mpl1K+FKeG33ef2AXQiH6vWqOtmc58B3gT9G0U4EngKmEgDiI6A+j4ba3hvwGNR9NUO6+hfAXZhVwe9akekKz4PrMCDP7enRcodPMPpc173HR+A7CXsLNwN+jae8G7nH337t7P/ANwhfT6wh/iU4Bvurufe7+AHA3cEH02n7gYDOrdfct7r50J1fvdnf/q7sPEELh8BHmfxuw0t1vdPeB6P1+SWgSG416vuTuPe5+H+FL/FZ33+jua4GHgSMA3H159Hn1unsTcA1w4g6Wu6PPOOPb7r7a3bsJYV8erUupu690938Ms+xbibaHmdUQ/nq+Nevz2NfMprt7h7s/tjMfhpklo9o/6+7t7r6SsGdwUTTLecC33H1N9Hv61Z1Zfpb3Aje4+1J37wU+CxxvZntH61BD2EM0d3/R3ddlrd/ubO+iplCYoMzsQsJfaH8AvpY1aQuhmWKPzAh3/4yH4wq/IvwlnfGYu0/1cExhFnAI8J/RtNmEZoHMMtKEv1LnRNNWR+MyVkXTAN5B+BJaZWZ/NLPjd3L11mc97iIE0I7sBRwbBVyLmbUQvlBmjVI9G7Ied+d4PgXAzGaY2SIzW2tmbcBPgOk7WO6OPuOM1VnTlwNXEPZmNkbvNXuYZd8CnGtm5cC5wFJ3z7zXBwl7KX8zs8Vm9rYd1JjLdMIe4qqscdnbf3Z23UMe74yhn08HoWluTvSHyHcJTY0bzOw6M6uNZt3d7V3UFAoTkJnNILSnfhj4F+A8M3sjgLt3Etrhz92ZZbr7BsJf12+PRr1K+LLNvKcRmjHWRtP2NLPs35950TTcfbG7n0VoYrkDuC3zNjtT005YDfwxCrjMMMXdPzLG9XwlWuZh7l4LXEho2sgY+n47+oxzvsbdb3H310evc7b9gyB7vhcIX6inA+8hhERm2t/d/QLC5/E14BdmVp3/atJM+Gt8r6xxg9uf0HwzN2vanjux7GxDP59qYBpbf8++7e5HEf6Y2R/4dDR+uO0teVAoTEzfBe5w9wejXebPAD+M/iokev4BM7syChDMbC4wf7gFmtk0wsHI56NRtwFvNbOTzawU+DfCwc5HCaHTCXzGzErN7CRCmCwyszIze6+Z1UVNIm2EZg8If2FPM7O6UfocMu4G9jezi6J6Ss3saDM7aIzrqSEcRG4xszlEX1JZNgD7ZD3f0We8HTM7wMzeHG3nHsJeSirXvJFbgMuBNxKOKWSWc6GZNUZ7Ji3R6GGXY2YV2QNhT/Q24OrodOi9gH8l7Bll1usTZjbHzKYSDo6PpHTI+5RE9b/fzA6P1vk/gcfdfWW0fY+NPrfO6PNIjbC9JR9xH9TQsHMDcDbhL6ipQ8bfD1yd9fxY4F7Cf/oW4DngamBaNP1iwn+WzJk3GwltzjOylnEO4YBhK/BHogO30bRDonGt0TznROPLgN8SmrHagMXA67NedwOhCaCF4c8++nLW85MY4eB0NN8BhDOWmqLlP0A4FrFT9ZD7QHNJ1vxryDqIT/gi/FzWZ/JE9Hk+RfiSX5M171nAK9F7fSqPz3gl2x6YPoxw7KedcDD+7lyfYdb88whf4PcMGf+TaHt3EP4IOHuY158Urf/QYV/CsaufRJ/3auDzbD37qISwJ7uJcPbRJwl7FjbM+6zM8R5fjqZdSjhonlnfudH4kwlnHHWw9UyvKSNtbw0jDxZ9wCIiBWFmpwM/cPe9RpxZYqfmIxEZVWZWaWZnmFlJ1Iz2BcJJDjIBaE9BJgQzewPwm1zTPJw9JeOEmVURmsIOJBz3uAf4hLu3xVqY5EWhICIig9R8JCIigyZcJ1vTp0/3vffeO+4yREQmlCeeeKLZ3RtHmm/ChcLee+/NkiVL4i5DRGRCMbNVI8+l5iMREcmiUBARkUEKBRERGaRQEBGRQQoFEREZpFAQEZFBCgURERlUNKGwqbWD515ajrr1EBEZXtGEwpPXnM2htxxFT3dX3KWIiIxbRRMKtYecAkBH66aYKxERGb+KJhRKqqYC0NW2OeZKRETGr+IJheooFNoVCiIiwymaUCifUg9Ab8eWmCsRERm/iiYUKmtCKAx0tsRciYjI+FU0oVBV0wDAQJdCQURkOEUTClOmTgPAu1tjrkREZPwqmlAor6xhwBN4r+4dLiIynKIJBUsk6LAqEj3aUxARGU7RhAJAl1WT7G+PuwwRkXGrqEKhOzGFUoWCiMiwiioUekuqKRvoiLsMEZFxq2ChYGY3mNlGM3tumOlmZt82s+Vm9oyZHVmoWjL6SmqoSGlPQURkOIXcU7gJOG0H008H9ouGS4DvF7AWAAZKa6hMdxb6bUREJqyChYK7/wnYUUdDZwE/9uAxYKqZ7VGoegBSZbVMcYWCiMhw4jymMAdYnfV8TTSucCrqqKaH/oGBgr6NiMhEFWcoWI5xOW+LZmaXmNkSM1vS1NS06+9YUUvCnM5WdYonIpJLnKGwBtgz6/lc4NVcM7r7de6+0N0XNjY27vIbJitD99kduqeCiEhOcYbCXcA/R2chHQe0uvu6Qr5h6T/uA6CrrbmQbyMiMmGVFGrBZnYrcBIw3czWAF8ASgHc/QfAvcAZwHKgC3h/oWrJqDjho/C7h+lu0y05RURyKVgouPsFI0x34GOFev9cKutCT6l9uvuaiEhORXVFc/XUGQAMdGhPQUQkl6IKhZr6cJA63aU9BRGRXIoqFMoqqun1UujW3ddERHIpqlDAjHabQrJX1ymIiORSXKEAdCRrKO3TjXZERHIpulDoSdZS3q9bcoqI5FJ0odBbWkdlSqEgIpJL0YVCf1kd1WndU0FEJJeiCwWvmEqtdxCunRMRkWxFFwpUNlBlvXR06racIiJDFV0oJKrrAWjbok7xRESGKrpQKKluAKCrZTfuyyAiMkkVXSiU10wH1H22iEguRRcKVXUhFHofvS7mSkRExp+iC4WahlkA9M97fcyViIiMP0UXCrXTQih4p5qPRESGKrpQSJZX0UUFqPtsEZHtFF0oALRaLaW9utGOiMhQRRkKnSVTKe9T99kiIkMVZSh0l9ZT1a8b7YiIDFWUodBf3kBNWvdUEBEZqihDIV05jXpvJZVKx12KiMi4UpShQPU0KqyfllY1IYmIZCvKUEhOaQSgbdP6mCsRERlfijIUyutmANCxRaEgIpKtKEOhqn4mAD0tG2OuRERkfCnKUMj0f9TXplAQEclWlKFQN20PAFLL7ou5EhGR8aUoQ6GkspZuyqFuTtyliIiMK0UZCpjRkqinrFt3XxMRyVbQUDCz08xsmZktN7Mrc0zfy8zuN7NnzOwhM5tbyHqytZc0UNmn7rNFRLIVLBTMLAlcC5wOHAxcYGYHD5ntG8CP3f0w4CrgK4WqZ6ie8unUDKj7bBGRbIXcUzgGWO7uK9y9D1gEnDVknoOB+6PHD+aYXjADVY3Up7fg7mP1liIi414hQ2EOsDrr+ZpoXLangXdEj88Basxs2tAFmdklZrbEzJY0NY3OcQCvnkW9ddDS1jEqyxMRmQwKGQqWY9zQP8s/BZxoZk8CJwJrgYHtXuR+nbsvdPeFjY2No1JcSV24gG1L86ujsjwRkcmgpIDLXgPsmfV8LrDNN7C7vwqcC2BmU4B3uPuY9GldUR+uVWjbuAYWHDAWbykiMu4Vck9hMbCfmc03szLgfOCu7BnMbLqZZWr4LHBDAevZRvW00JLVvWXtWL2liMi4V7BQcPcB4DLgd8CLwG3u/ryZXWVmZ0aznQQsM7OXgJnA1YWqZ6j6GeHs1/5WdYonIpJRyOYj3P1e4N4h4z6f9fgXwC8KWcNwqupD/0fp9g1xvL2IyLhUnFc0A1ZSTotPoWT9U3GXIiIybhR0T2G8aymdQUWiaHNRRGQ7Rf2N2Fkxk9p+9X8kIpJR1KHQXzWLaelNpNO6qllEBIo8FKjdg2nWRnNrW9yViIiMC0UdCiVTw2mpm9avirkSEZHxoahDoXr6PADaN7wScyUiIuNDUYdC3ay9AOjZvHqEOUVEikNRh8LUGWFPIdWiri5ERKDIQyFRWUcXFSQ61NWFiAgUeShgxmZqKdu8LO5KRETGheIOBaCzeh611h13GSIi40LRh0Jv9RxmpDcykErHXYqISOyKPhSYOo9Ga2X9pi1xVyIiEruiD4WK6XsD0Lx2RbyFiIiMA0UfCjWz9gGgfb1CQUSk6EOhYc4CAHo3rYy3EBGRcaDoQ6F86hwGSECLrmoWERkxFMxsgZmVR49PMrPLzWxq4UsbI8kSNiemU975atyViIjELp89hV8CKTPbF/gRMB+4paBVjbG28j2o7VEoiIjkEwppdx8AzgH+x90/CexR2LLGVm9/PzPS6+kdSMVdiohIrPIJhX4zuwB4H3B3NK60cCWNPdv3ZPawLaxt2hx3KSIisconFN4PHA9c7e4vm9l84CeFLWtsVc7cF4CmV9QHkogUt5KRZnD3F4DLAcysHqhx968WurCxVD/3QADa174EvD7eYkREYpTP2UcPmVmtmTUATwM3mtk1hS9t7NTNOQCAgU26gE1Eils+zUd17t4GnAvc6O5HAf9U2LLGllXV025TKG1dGXcpIiKxyicUSsxsD+A8th5onnQ2lc2ltkv3ahaR4pZPKFwF/A74h7svNrN9gL8Xtqyx9/f+6cwceJW+AXWhLSLFa8RQcPefu/th7v6R6PkKd39H4UsbW3uWtTPHmlm1UaelikjxyudA81wz+5WZbTSzDWb2SzObOxbFjaXq4z9I0px1L78QdykiIrHJp/noRuAuYDYwB/h1NG5EZnaamS0zs+VmdmWO6fPM7EEze9LMnjGzM3am+NE0ff6hALSvUSiISPHKJxQa3f1Gdx+IhpuAxpFeZGZJ4FrgdOBg4AIzO3jIbJ8DbnP3I4Dzge/tVPWjqHJWuFbBm3QBm4gUr3xCodnMLjSzZDRcCGzK43XHAMujYxB9wCLgrCHzOFAbPa4D4uuVrqya5uQMqtp0rYKIFK98QuEDhNNR1wPrgHcSur4YyRwg+yYFa6Jx2b4IXGhma4B7gY/nWpCZXWJmS8xsSVNTUx5vvWtaq+czo/cVUmkv2HuIiIxn+Zx99Iq7n+nuje4+w93PJlzINhLLtbghzy8AbnL3ucAZwM1mtl1N7n6duy9094WNjSO2XO2yJ7tnMJ+1rN7UUbD3EBEZz3b1zmv/msc8a4A9s57PZfvmoQ8CtwG4+1+ACmD6Lta0246rWE219bJyhY4riEhx2tVQyLUXMNRiYD8zm29mZYQDyXcNmecV4GQAMzuIEAqFax8aQeM5XwGgZeXTcZUgIhKrXQ2FERvdoxvzXEa4GvpFwllGz5vZVWZ2ZjTbvwEfNrOngVuBi909tgb98tnhtFRf/1xcJYiIxGrYrrPNrJ3cX/4GVOazcHe/l3AAOXvc57MevwCckFelY6GilubSPahteynuSkREYjFsKLh7zVgWMl601e7PvKbltPf0U1MxqW4wJyIyol1tPpq0Hu2YxT72Kn9b0xx3KSIiY06hMMTbq18kac6aZU/EXYqIyJhTKAxRd1G4/XTvqsUxVyIiMvYUCkNNnUdbsp665qfirkREZMzl03V2u5m1DRlWR91p7zMWRY4pMzbXv5b9B5bR1N4bdzUiImMqnz2Fa4BPE/otmgt8CvghoYO7GwpXWnyS845hQWIdzy1fGXcpIiJjKp9QOM3d/5+7t7t7m7tfB5zh7j8D6gtcXyxmHhQundjw4iMxVyIiMrbyCYW0mZ1nZoloOC9r2qTsTrRs3lGkMfyl38ZdiojImMonFN4LXARsjIaLCN1dVxK6sZh8ymvYVLUvs9PraOvpj7saEZExk0/X2Svc/e3uPj0a3u7uy929293/PBZFxiE15ygOt+UseVkXsYlI8cjn7KO50ZlGG81sg5n90szmjkVxcZp2wAnUWRfX3X5f3KWIiIyZfJqPbiR0eT2bcAbSr6Nxk1rpvGMAOL5ct+cUkeKRTyg0uvuN7j4QDTcBhbv92XgxfX+6qWB2y1IdVxCRopFPKDSb2YVmloyGC4FNhS4sdokEXXu+keMSL7Jk5ea4qxERGRP5hMIHgPOA9cA64J3A+wtZ1HhRe9DJ7Jlo4oUXno27FBGRMZHP2UevuPuZ7t7o7jPc/Wzg3DGoLXal+54EQO/fH4q1DhGRsbKrHeL966hWMV41HkCXVbFvxxOs3twVdzUiIgW3q6Fgo1rFeGVGav+38sbEMzz44rq4qxERKbhdDYVJ2b1FLjWHnka9dbDi6YfjLkVEpOCGDYVhusxuM7N2wjULxWHBm0mTYNq6P9LS1Rd3NSIiBTVsKLh7jbvX5hhq3L1kLIuMVVUD3aVTOcmWct8LG+KuRkSkoHTntTxUnXgFr0ms5PEndTc2EZncFAp5sIPeBsDUVb9jY1tPzNWIiBSOQiEf0xbQV1LDqYm/8vMn1sRdjYhIwSgU8lRWXc8xiWU8+PhS0umiOflKRIqMQiFf77sTgKPaH+AvKyZ/108iUpwUCvlq2If0nIWcW/ooixavjrsaEZGCUCjshET3Fg5gFa8+/wibOnrjLkdEZNQVNBTM7DQzW2Zmy83syhzTv2lmT0XDS2bWUsh6dtslD5EuqeJ8+x03Pboy7mpEREZdwULBzJLAtcDpwMHABWZ2cPY87v5Jdz/c3Q8HvgPcXqh6RkVFLYmKWs5KPsqvHn2Wdt18R0QmmULuKRwDLHf3Fe7eBywCztrB/BcAtxawntFRswdlDHBa/wOc8S31hyQik0shQ2EOkH1Edk00bjtmthcwH3iggPWMjn95COYdz4crH6Snr5+e/lTcFYmIjJpChkKu7rWHO8H/fOAX7p7zG9bMLjGzJWa2pKmpadQK3GVHf4iZA69yaPdiTvnmH+OuRkRk1BQyFNYAe2Y9nwu8Osy857ODpiN3v87dF7r7wsbGxlEscRcdfBZeN5f/mHIPWzr7aGrXmUgiMjkUMhQWA/uZ2XwzKyN88d81dCYzOwCoB/5SwFpGV7IUO+EK9ut7gdf2P8UZ3/pT3BWJiIyKgoWCuw8AlwG/A14EbnP3583sKjM7M2vWC4BF7j6x+o444kJIlvGN8h+xqaOHJ1/ZEndFIiK7zSbad/HChQt9yZIlcZcRPPsL+OUHuar/IhYl38bTXziF0qSuBxSR8cfMnnD3hSPNp2+w3XHoO6C8ln8v+znJvnau+9OKuCsSEdktCoXdYQbvu4ty7+GrZT/iG79bxuKVm+OuSkRklykUdtfsI6B6JmckHuOAkvW854eP6UY8IjJhKRRGw6UPY5bgl1VfoSRhfOSnS3VRm4hMSAqF0VAzE+r3orqvicurfs8Tq7bwiUVPMpBKx12ZiMhOUSiMlo8vhcoGPtJ7A6fUvcrvnt/Af/zqWSba2V0iUtwUCqPFDD7+BCTLuC71OT7zxhnctmQNx1z9B/q1xyAiE4RCYTRVNUDjATDQy0c2fIFPvmkvmjr6WPjlP9DZOxB3dSIiI1IojLZLH4bp+2OrHuETnd/hK+ccSmt3Pwu//AeWb2yPuzoRkR1SKBTCZX+FN30OnlnEBQ+fys0fWEh/Ks0p3/wTdzy5Nu7qRESGpVAolDd+Cl53OXSs4w13nsAjn34D1eUlXPGzp/jCnc/R3adTVkVk/FEoFIoZnPIlOPkL0NnEzOtew5NXvp5ZteX8719WcfhV9/Ho8ua4qxQR2YZCodDe8K/QsAC6t1DyzQN57PLXcuCsGgDec/3jHP3l37O5sy/mIkVEAoXCWLh8KbzrJujrgG8ewm/Pb+DpL5zCpScuoKmjjyO/9HtO+OoDtPf0x12piBQ5dZ09ltYuhRtOhfQAnHczHPQ2lm9s510/+AtbukIgzKot5+eXvo49G6piLlZEJpN8u85WKIy1tnXw3YVhr+H4y+Dkz0NJOc+uaeXiG//Kpqgpqb6qlO9feBTHzm/ALNftrkVE8qdQGM/6u+G+z8Hi66GsGj74e5h5CADrW3s493uPsK61BweqypLMqq3gN1e8gfKSZLx1i8iEpVCYCP52D9z2z6E5qXYOfOxxKA8Hobv7Utzx1Fqu+vULdPenMGD21Epu/+jrmFlbEW/dIjLhKBQmiq7N8L3joGMDJErhLVfBwg9Aafjid3f+vLyZj9/yJC3d4bhD0mBufRWL/uU49qirjLN6EZkgFAoTzZolcPO50NsKyTI447/g8PdCsnRwlpebO7no+sd5tbWbdLTZppSX0FBdSkN1GXd87PUxFS8i451CYaJa8RAsuhD62qGkAs78brgXdGLbs4fP/M6f+duGdkoSRld0dXTCIJkw9plezZ2XvZ6KUh2DEJFAoTCRucP3joeWVdDfBaVV8I7r4YAzwpXSQ6xs7uSiHz3Oq609pNJbt2dJwpg9tZJrznsth86pU0iIFDGFwmSQTsPzt8OdH4OBHiibEi6CW3DydnsOGd19Kc6+9s+saO5kIOVkb93MnkTSjP1m1nDnx04gkdDpriLFQKEwmaQG4Olb4J5PQaoXSirh1KvhtRdA2Y4vctvY1sN7r3+cVZs6STnb7ElACIkZNeVMKS+hqizJ7R89gaSCQmTSUShMRgN98MIdcPcnw8VviSQcfiEc9m6Yd/ywew/Z0mnn7Gsf4aWN7aTSTirtuJNzjyJhxvzp1dz4/qOZWVOhvQqRCUyhMJm5w+rH4WcXQtcm8DQky+H4j4aAmHHQTi2upz/FO773KP9o6iDlTjrHHkXCoKI0Sd9AmkR0Sux/vesw9p5WTUN1ma66FhnnFArFoq8T/nYv3Ptp6NkSxpVWw5s+C4e+E2r32KXFptPOud9/hGXr20l76Hajpz9Na08/uX5lqsuTVJQkae8dYF59Jd989xHMqa9kamWp9jBExgGFQjHqaAoHpu+/KjQvAcw/Mew9HPR2qKgdlbcZSKVZs6Wbf7l5CSs3dZF2p7I0Sc9Amr6B9HbzG+GkqdqKUkqTRmt3P2bGnvWVfPmc1zB9SjmNNeXUVpRoj0OkQBQKxW7TP+DHZ0FnUzhzCaBqOpz13XD2UklZQd62pz/Fqk1dXHbLUl7Z3IV7OAMq7VBRkqAvlaY/lft3zgxKEwkG0mnMjIRlAsWY11DFf577GhqjAKkuLylI/SKTlUJBAndY+wQ88zNYciOk+8ESsO8/wT4nwT5vCscgxvAv9HTaae3u559veJzlGzvwqMyG6jL6U2m2dPWHMBlyAHwoi0IjYRYem2HAvIYqvvbOw6irLKW2opTayhJ1JihFb1yEgpmdBnwLSALXu/tXc8xzHvBFwv//p939PTtapkJhN6T6wxXTf78Plt4MA91h/JSZWwNin5N2+ThEIaTSzubOPprae/nkz56iP5WmP5VmfVvPYGjkEyCwtRmrsixJSSJBZ+/AYLBgxuy6Cj75lv2pjcKkrrKEmorwuKI0oaYtmdBiDwUzSwIvAW8B1gCLgQvc/YWsefYDbgPe7O5bzGyGu2/c0XIVCqOoZXUIiRUPwQt3hr0ICFdQH/k+WPAm2Ot1gz23jnf9qTSbOkKANHf08uV7XmDNlu7B4ICwR1JdnmQg7aF7kDzCJMOifwyoKishmTA6ewfCtKxwyUTH3PpKPnv6QVSVJ6kqC9eBVJYmqY6uCSkvUdDI2BkPoXA88EV3PzV6/lkAd/9K1jxfB15y9+vzXa5CoUDSadjwXAiIh/8belq2TiuvDTcEmncszD5y1A5Yjyc9/Snaevpp6x6graef9p4BvvTr51kdhQrRsZHMf5eqsiQpd7r7UoPjduV/UuYK84F0OECfCYnMXs206jISZjR39pEdH5ksMWCvadV8+exDKU0mKE0mKCuxwcelyQRlyQSlJUZ5SVIXJhax8RAK7wROc/cPRc8vAo5198uy5rmDsDdxAqGJ6Yvu/tscy7q0uUyZAAAOXElEQVQEuARg3rx5R61ataogNUuW/p5wLcQdH4X2V8O1EBmlVfCad8Gex8Dco2HafnldOFcM+lNpuvpSdPUN0Nm79Wd3/wBdfSmuue8l1rZsDZr66jLSaQ/HUaJxThQwHr7882ka21mW9cAI16AkLHSTQlYwZb+goaqMhMHmzr7t5jGDuVMrMTPWbOka3GNa0FjNt84/gvLSJGVRQJUkM6Fl2lMaQ+MhFN4FnDokFI5x949nzXM30A+cB8wFHgYOdfeWHIsEtKcQm+6WcMB6zWJ47Adbr4nIqJgKx14Kex4Nc46Cyvp46pykBlJpegfS9PSnBn9+YtGT/KOpk70aqnBg1abObcIj8197Zm0F7s6G9t7B0Bk6z5TyEtLudPYObBdAmYBKJgx3SBXqOwMoSYYr6ftSaYbGRaZ5rrosiZnR0Tuw3TyZkKuvKsMyAcbQgAtduyTM2NDWM9gsmPmx17Rq/u+Zh4Sr+qMr+5PRyQwJs+hqf7ablukFIJFgcPw2zxNZy8g6OWKsjIdQyKf56AfAY+5+U/T8fuBKd1883HIVCuNEOg2bloeQuP8q6G0LPbpmWCKcAnvSv8Osw2DGwVA+Jb56ZVSl005fKk1vf5regRBUvQMprlj0FCuaO7cew8m8INrbmVVbgeOsb+uNxm8bUlMry3Cclq5+sl66lTuVZSW4O939qe3niR4kE4az/ZX541EmlIZ+FWfioiQZHg2knH0aq7n/307atffJMxQKebL3YmA/M5sPrAXOB4aeWXQHcAFwk5lNB/YHVhSwJhktiQQ07h+GI94bxvW2w9qlISj+cm24RuKef9v6mpIK2P80mHUozHwNzHoN1M4e09NhZXQkEkZFIhl1x771RlB3X/6G+IrKg3vo72sgK9TCz9Tg8ytvf4aVzZ3Ma6gGnFc2hz92hsbLnKnhrodrW7qHPWEh3DrX2dDWu/30KAWmTynHgeaO3pw1T60qA4ct3X2UJgvfTFvoU1LPAP6HcLzgBne/2syuApa4+10W9p3+GzgNSAFXu/uiHS1TewoTiDu0roH1z8Jv/h36O0O3HJmL6QCwcCD7sHfB9AOgMRqmzFRYiIyi2JuPCkWhMAn0tMHGF0JYrH8GnvtVuNPcUOU1cPDZUVAcCNP3h7o9dVBbZBcoFGRicYf29dC8DJqi4dmfhyap7B1vS4Rmp0xINB4YQqN+PiTV9YXIcMbDMQWR/JmFK6lr9whXVQO87Zrws2tzFBR/g+aX4KlbYN0zbNeKW1oF+50SQmL6/jBtX2iYDxV1Y7giIhOb9hRk4uptDyHRtAweuBo61ofrKXxoT60WnQ3VAEd/CBr22TpU1uvYhRQFNR9J8ervCafLbl4Rhse+B53NUVjk+H23BFQ2wFHvC81Q9XuHoXZ2uLudyCSgUBDJpb8HWlZFgfEyPPpt6Ng4TGBYtBdhUN0Ix38shEXDfJi616Ts7kMmL4WCyM5KDUDbGtiycuuw9Gbo3pyjSQqwJKEviuhCvWM+BLVzoW4u1M2B2jlQUj626yAyDIWCyGjr3gJbVm0NjMd/EC7Q83R0IVKO/0uJUvBUCI7qRjjuo1FozA2hUTNLTVQyJhQKImOtvxvaXoXW1dC6FtrWwuLrw/EMPPfeBkCyHNID4aD34e8J12Jk9jTq5kLVNB0Ml92mUBAZb9yhpzWEReva0FTVugaW/jicdrujA+EQLuY74K1ZzVNZzVQ6viEjUCiITETpNHQ1h7BoWxt+Pvod6NgQ9aE9XHDkOr4xB6bMgCmzQjNVZYOuBi9iCgWRySrVH67+zoRG65r8jm8Ag2dUWSI0Sx35PqiZGfqamjIrCpGZUFoxlmskY0ChIFLMMsc3OjaGvYyODfDIt6M9jmhvY4fhQdRsZWEP47XvjoJj5tbgmDIzHAfR3seEoFAQkZGlBkJzVfv6reHRvgEW/wi6mnbcZDUo2vuoqIOD3p4VHkNCpKxqrNZKclDfRyIysmRJON5QM2vb8Sd+etvn7qFbkew9j8zjJ38CXZsg1QdP3xqat4ZTUgnJUkiWhVu6ZgKjJitEqqbpNN0YaU9BREZXOhVCIrPXkQmRv14XAiPVB30dw5+iCwzufZTXwYFvDaFRPSMERvW08LNqevip4x95UfORiIx/fZ3b731kmrL+dnc4hXekYx/ANk1Y+58Whcf0EByDP6MgKa8pyus+FAoiMnmk0+GK8q5N4RhI16ZwUWDXpnCNR1czvHQf9EYhYrbjPZFkWbjaPFkK+71l29DYJkymQcXUSXEwXccURGTySCTCl3b1NMKt3PPQ1xkFRzN0RmGS/XzZPeEugM/+grz3RLBwxtUh52wNjappoVv2qmnhTK2qBiit3L31jZFCQUQmp7LqMNTvNcwM1277tL9n655IZ/beSPPWg+nuoYPExT/MowALZ1wlSiFRAoeeG4VGFCKV9dsGSmnVuGjWUvORiMiuSPWHpqvuzVnNWJui55vhmdsg3R9O++3vHOHAekbmAHsN7POmbUOjahrMXRhuDrUL1HwkIlJIydJwVlTNzNzTT716+3GpAehpyREi0fOnF4XnvR3w4l3bH2RvWACXLy3I6mQoFERExkqyJByLqJ6ee/opX9p+XDoVzsLq2hz2GgpMoSAiMp4lklETUuEDAWDin2clIiKjRqEgIiKDFAoiIjJIoSAiIoMUCiIiMkihICIigxQKIiIySKEgIiKDJlzfR2bWBKzaxZdPB5pHsZzxaLKv42RfP9A6Tgbjcf32cvfGkWaacKGwO8xsST4dQk1kk30dJ/v6gdZxMpjI66fmIxERGaRQEBGRQcUWCtfFXcAYmOzrONnXD7SOk8GEXb+iOqYgIiI7Vmx7CiIisgMKBRERGVQ0oWBmp5nZMjNbbmZXxl3PaDCzlWb2rJk9ZWZLonENZvZ7M/t79LM+7jp3hpndYGYbzey5rHE518mCb0fb9BkzOzK+yvM3zDp+0czWRtvyKTM7I2vaZ6N1XGZmp8ZTdf7MbE8ze9DMXjSz583sE9H4SbEdd7B+k2MbuvukH4Ak8A9gH6AMeBo4OO66RmG9VgLTh4z7OnBl9PhK4Gtx17mT6/RG4EjguZHWCTgD+A1gwHHA43HXvxvr+EXgUznmPTj6fS0H5ke/x8m412GE9dsDODJ6XAO8FK3HpNiOO1i/SbENi2VP4RhgubuvcPc+YBFwVsw1FcpZwP9Gj/8XODvGWnaau/8J2Dxk9HDrdBbwYw8eA6aa2R5jU+muG2Ydh3MWsMjde939ZWA54fd53HL3de6+NHrcDrwIzGGSbMcdrN9wJtQ2LJZQmAOsznq+hh1vxInCgfvM7AkzuyQaN9Pd10H45QVmxFbd6BlunSbbdr0saj65IavZb0Kvo5ntDRwBPM4k3I5D1g8mwTYsllCwHOMmw7m4J7j7kcDpwMfM7I1xFzTGJtN2/T6wADgcWAf8dzR+wq6jmU0Bfglc4e5tO5o1x7hxv4451m9SbMNiCYU1wJ5Zz+cCr8ZUy6hx91ejnxuBXxF2STdkdr2jnxvjq3DUDLdOk2a7uvsGd0+5exr4IVubFybkOppZKeEL86fufns0etJsx1zrN1m2YbGEwmJgPzObb2ZlwPnAXTHXtFvMrNrMajKPgVOA5wjr9b5otvcBd8ZT4agabp3uAv45OnvlOKA10zwx0QxpQz+HsC0hrOP5ZlZuZvOB/YC/jnV9O8PMDPgR8KK7X5M1aVJsx+HWb9Jsw7iPdI/VQDjD4SXCkf//E3c9o7A++xDOaHgaeD6zTsA04H7g79HPhrhr3cn1upWw691P+Avrg8OtE2G3/Npomz4LLIy7/t1Yx5ujdXiG8CWyR9b8/ydax2XA6XHXn8f6vZ7QPPIM8FQ0nDFZtuMO1m9SbEN1cyEiIoOKpflIRETyoFAQEZFBCgURERmkUBARkUEKBRERGaRQEBGRQQoFkTyY2eFDukI+c7S6YDezK8ysajSWJbK7dJ2CSB7M7GLCRVWXFWDZK6NlN+/Ea5LunhrtWkS0pyCTipntHd385IfRDVDuM7PKYeZdYGa/jXqZfdjMDozGv8vMnjOzp83sT1HXKFcB745unvJuM7vYzL4bzX+TmX0/uvHKCjM7Meol80Uzuynr/b5vZkuiuv5vNO5yYDbwoJk9GI27wMLNk54zs69lvb7DzK4ys8eB483sq2b2QtQr5zcK84lK0Yn7kmoNGkZzAPYGBoDDo+e3ARcOM+/9wH7R42OBB6LHzwJzosdTo58XA9/Neu3gc+Amwj06jNB3fhvwGsIfXU9k1ZLp1iEJPAQcFj1fSXSzJEJAvAI0AiXAA8DZ0TQHzsssi9BlgmXXqUHD7g7aU5DJ6GV3fyp6/AQhKLYRdXv8OuDnZvYU8P8Id9QCeAS4ycw+TPgCz8ev3d0JgbLB3Z/10Fvm81nvf56ZLQWeBA4h3JFrqKOBh9y9yd0HgJ8S7tQGkCL0zAkheHqA683sXKArzzpFdqgk7gJECqA363EKyNV8lABa3P3woRPc/VIzOxZ4K/CUmW03zw7eMz3k/dNASdQ75qeAo919S9SsVJFjObn63s/o8eg4grsPmNkxwMmEXn8vA96cR50iO6Q9BSlKHm6K8rKZvQsGbx7/2ujxAnd/3N0/DzQT+sJvJ9yPd1fVAp1Aq5nNJNwYKSN72Y8DJ5rZdDNLAhcAfxy6sGhPp87d7wWuINzYRWS3aU9Bitl7ge+b2eeAUsJxgaeB/zKz/Qh/td8fjXsFuDJqavrKzr6Ruz9tZk8SmpNWEJqoMq4DfmNm69z9TWb2WeDB6P3vdfdc98SoAe40s4povk/ubE0iueiUVBERGaTmIxERGaTmI5n0zOxa4IQho7/l7jfGUY/IeKbmIxERGaTmIxERGaRQEBGRQQoFEREZpFAQEZFB/x81gvj5U7FenAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f761c292a20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#调整max_depth和min_child_weight之后再次调整n_estimators(6,4)\n",
    "xgb2_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=270,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=4,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit_1(xgb2_3, X_train, y_train, cv_folds = kfold)\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#results = cross_val_score(xgb2_3, X_train, y_train, metrics='mlogloss', cv=kfold)\n",
    "#print results\n",
    "#print(\"CV logloss: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第四步：调整树的参数：subsample 和 colsample_bytree\n",
    "(粗调，参数的步长为0.1；下一步是在粗调最佳参数周围，将步长降为0.05，进行精细调整)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.6, 0.7, 0.8, 0.9],\n",
       " 'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8]}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59173, std: 0.00319, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.59103, std: 0.00360, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.58999, std: 0.00372, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.58969, std: 0.00387, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58913, std: 0.00361, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58916, std: 0.00338, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.59148, std: 0.00371, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.58966, std: 0.00375, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.58898, std: 0.00348, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.58865, std: 0.00393, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58839, std: 0.00380, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58833, std: 0.00354, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.59142, std: 0.00395, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.58981, std: 0.00339, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.58935, std: 0.00380, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.58817, std: 0.00404, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58815, std: 0.00353, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58807, std: 0.00361, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.59088, std: 0.00333, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.58896, std: 0.00374, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.58815, std: 0.00382, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.58796, std: 0.00320, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.58789, std: 0.00382, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.58770, std: 0.00306, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       " -0.5876973362313377)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=270,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=4,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_,     gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([124.07171383, 105.66965942, 114.18845601, 112.02012   ,\n",
       "         88.50554218, 114.76518912, 108.52348127, 121.83029146,\n",
       "        133.64124351, 120.02896824, 113.60965981, 107.76050963,\n",
       "        113.19117956, 134.23949699, 144.08447547, 139.5760993 ,\n",
       "        129.61468801, 111.79613371, 130.47664132, 149.08588538,\n",
       "        147.12337313, 152.87393208, 135.63924356, 128.34596386]),\n",
       " 'mean_score_time': array([4.47126174, 0.31437607, 0.39291649, 0.28649678, 0.28460445,\n",
       "        0.28371296, 0.61993413, 0.30126863, 0.80548558, 2.66364627,\n",
       "        0.54860988, 0.28629203, 0.28391356, 0.30566616, 0.3455359 ,\n",
       "        0.52568383, 0.37671862, 2.53665595, 0.29185848, 0.28340659,\n",
       "        0.28160043, 0.2776123 , 0.27791533, 2.52959681]),\n",
       " 'mean_test_score': array([-0.59173315, -0.59102763, -0.58998551, -0.58969418, -0.58913138,\n",
       "        -0.58915883, -0.59147906, -0.58965989, -0.58897682, -0.58865075,\n",
       "        -0.58838518, -0.58832779, -0.5914235 , -0.58980867, -0.58935046,\n",
       "        -0.58816763, -0.58815268, -0.58806526, -0.59088295, -0.58895837,\n",
       "        -0.58815321, -0.58795677, -0.58788838, -0.58769734]),\n",
       " 'mean_train_score': array([-0.54329159, -0.54151883, -0.54032171, -0.54035194, -0.53993148,\n",
       "        -0.54073769, -0.54137671, -0.53910036, -0.53834013, -0.53754393,\n",
       "        -0.53812864, -0.53820422, -0.53949433, -0.53721844, -0.53675333,\n",
       "        -0.53626427, -0.53647438, -0.53707073, -0.53789391, -0.5362855 ,\n",
       "        -0.53520885, -0.53539885, -0.53515681, -0.5358768 ]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.7, 0.7, 0.7, 0.7, 0.7,\n",
       "                    0.7, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.9, 0.9, 0.9, 0.9,\n",
       "                    0.9, 0.9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6, 0.7,\n",
       "                    0.8, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6,\n",
       "                    0.7, 0.8],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " 'rank_test_score': array([24, 21, 19, 17, 13, 14, 23, 16, 12, 10,  9,  8, 22, 18, 15,  7,  5,\n",
       "         4, 20, 11,  6,  3,  2,  1], dtype=int32),\n",
       " 'split0_test_score': array([-0.58662275, -0.58526073, -0.58395027, -0.58339235, -0.58310013,\n",
       "        -0.58378347, -0.58581479, -0.58328236, -0.58348645, -0.5822699 ,\n",
       "        -0.5817598 , -0.58266978, -0.58495971, -0.5842968 , -0.58361291,\n",
       "        -0.58227931, -0.58216906, -0.58206407, -0.58520613, -0.58308195,\n",
       "        -0.58166311, -0.58241921, -0.58176274, -0.58302646]),\n",
       " 'split0_train_score': array([-0.54449689, -0.54235902, -0.54173431, -0.54191523, -0.54117878,\n",
       "        -0.54247176, -0.54277103, -0.54045656, -0.54007235, -0.53873283,\n",
       "        -0.53904481, -0.53907328, -0.53965962, -0.53856457, -0.53785036,\n",
       "        -0.53827632, -0.53813008, -0.53847368, -0.53943345, -0.53725788,\n",
       "        -0.53741017, -0.53538208, -0.53553555, -0.53729649]),\n",
       " 'split1_test_score': array([-0.59019658, -0.58901683, -0.5883852 , -0.58763905, -0.58759593,\n",
       "        -0.58759236, -0.58856682, -0.5879856 , -0.58687893, -0.58739832,\n",
       "        -0.58711354, -0.58702287, -0.59052951, -0.58832189, -0.58714207,\n",
       "        -0.5867044 , -0.58713225, -0.5869763 , -0.58902567, -0.58629439,\n",
       "        -0.58669172, -0.58707205, -0.58590723, -0.5866776 ]),\n",
       " 'split1_train_score': array([-0.54382525, -0.54223796, -0.54046941, -0.54079328, -0.54012987,\n",
       "        -0.54081308, -0.54134523, -0.53922097, -0.53829767, -0.53789872,\n",
       "        -0.53855853, -0.53841612, -0.54014825, -0.53777999, -0.53680791,\n",
       "        -0.53676786, -0.53671584, -0.53666858, -0.5394232 , -0.53608051,\n",
       "        -0.53518979, -0.5359232 , -0.53463643, -0.53618455]),\n",
       " 'split2_test_score': array([-0.59169072, -0.59140823, -0.58986307, -0.59027026, -0.5894841 ,\n",
       "        -0.58907633, -0.59273566, -0.59053872, -0.58942915, -0.58798504,\n",
       "        -0.58911124, -0.58772614, -0.59050858, -0.5897767 , -0.58901936,\n",
       "        -0.58647653, -0.58805167, -0.58774031, -0.59298843, -0.59024189,\n",
       "        -0.58864292, -0.5881206 , -0.58804622, -0.58664413]),\n",
       " 'split2_train_score': array([-0.54341337, -0.54136233, -0.53986367, -0.53971937, -0.53925794,\n",
       "        -0.54069226, -0.54152385, -0.53807983, -0.53864919, -0.53695395,\n",
       "        -0.53785365, -0.53740808, -0.53905037, -0.53670129, -0.53645674,\n",
       "        -0.53518094, -0.53593383, -0.53704671, -0.53790375, -0.53723128,\n",
       "        -0.53490666, -0.53604076, -0.53537818, -0.53517426]),\n",
       " 'split3_test_score': array([-0.59506692, -0.5949847 , -0.5935541 , -0.59404591, -0.59315551,\n",
       "        -0.59339052, -0.59502343, -0.59357618, -0.59293061, -0.59313997,\n",
       "        -0.59158346, -0.59219515, -0.59505348, -0.59321412, -0.59351525,\n",
       "        -0.59342405, -0.59189061, -0.59131329, -0.59302875, -0.59292792,\n",
       "        -0.59130431, -0.5911318 , -0.59196404, -0.59079027]),\n",
       " 'split3_train_score': array([-0.54256253, -0.54091729, -0.53967923, -0.53968593, -0.5398384 ,\n",
       "        -0.54039227, -0.54028861, -0.53867181, -0.53812675, -0.53728418,\n",
       "        -0.53774325, -0.53864906, -0.5397576 , -0.5373742 , -0.53726256,\n",
       "        -0.53569914, -0.53582321, -0.53663106, -0.53662881, -0.53599773,\n",
       "        -0.53475557, -0.53455942, -0.53490789, -0.5353341 ]),\n",
       " 'split4_test_score': array([-0.59508982, -0.5944687 , -0.59417616, -0.59312435, -0.59232222,\n",
       "        -0.5919523 , -0.59525573, -0.59291756, -0.59215994, -0.5924617 ,\n",
       "        -0.59235907, -0.59202615, -0.59606763, -0.59343494, -0.59346393,\n",
       "        -0.59195503, -0.59152083, -0.59223358, -0.59416676, -0.59224671,\n",
       "        -0.59246527, -0.59104113, -0.59176286, -0.59134933]),\n",
       " 'split4_train_score': array([-0.5421599 , -0.54071752, -0.53986195, -0.53964591, -0.53925238,\n",
       "        -0.5393191 , -0.5409548 , -0.53907265, -0.5365547 , -0.53685   ,\n",
       "        -0.53744297, -0.53747453, -0.53885582, -0.53567212, -0.53538908,\n",
       "        -0.53539707, -0.53576896, -0.53653362, -0.53608034, -0.53486009,\n",
       "        -0.53378208, -0.53508878, -0.53532603, -0.53539459]),\n",
       " 'std_fit_time': array([13.81331004,  4.08844101,  5.98838581,  6.0845383 , 22.26935817,\n",
       "        15.88292291,  4.0871538 ,  5.86116545,  5.67288714,  6.31573672,\n",
       "         5.61594229,  5.95056195,  4.36030999,  4.06035861,  5.52459587,\n",
       "         6.43297869,  0.68493559,  4.67636091,  0.90618193,  5.40806683,\n",
       "        10.24545557,  4.33179669,  7.14618451,  7.2174792 ]),\n",
       " 'std_score_time': array([4.98451853e+00, 4.26966154e-02, 8.73255136e-02, 2.37131742e-03,\n",
       "        7.04579960e-03, 6.29323397e-03, 5.15708146e-01, 3.34581130e-02,\n",
       "        4.79753787e-01, 4.57661670e+00, 3.30655892e-01, 1.64552741e-02,\n",
       "        4.98559251e-03, 6.07434783e-02, 6.24195700e-02, 3.26800526e-01,\n",
       "        1.51352494e-01, 4.47760209e+00, 1.03333231e-02, 5.74465268e-03,\n",
       "        3.34398745e-03, 2.37786559e-03, 5.38797895e-03, 4.50295692e+00]),\n",
       " 'std_test_score': array([0.00318935, 0.00360403, 0.00372278, 0.00387229, 0.00361203,\n",
       "        0.00337837, 0.00371402, 0.00374698, 0.00347894, 0.00393455,\n",
       "        0.00379632, 0.00354152, 0.00395373, 0.0033864 , 0.00379925,\n",
       "        0.00404099, 0.00352625, 0.0036119 , 0.00333157, 0.00373759,\n",
       "        0.0038215 , 0.00319589, 0.0038235 , 0.00306164]),\n",
       " 'std_train_score': array([0.00084435, 0.00067104, 0.00075514, 0.00089228, 0.00070973,\n",
       "        0.00101454, 0.00081593, 0.00078482, 0.00112527, 0.00069806,\n",
       "        0.00058625, 0.00065793, 0.00047519, 0.00098084, 0.00082643,\n",
       "        0.00114429, 0.00089593, 0.00072282, 0.00138554, 0.00089405,\n",
       "        0.00119822, 0.00054554, 0.00033282, 0.00079131])}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.587697 using {'colsample_bytree': 0.9, 'subsample': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzsnXl4Tdf3h9+dWUQSSYgQSYRUkck81RAxlpiHUi01VVst9a0fqkqVllLValVNpaWltGqseaaGIKYgM4khEjEkITLt3x/3SoOESO51M+z3ee7z5J5z9trrnCT3c/fea68lpJQoFAqFQqFrjAztgEKhUCiKJ0pgFAqFQqEXlMAoFAqFQi8ogVEoFAqFXlACo1AoFAq9oARGoVAoFHpBCYxCoVAo9IISGIVCoVDoBSUwCoVCodALJoZ2wJA4ODhINzc3Q7uhUCgURYrjx4/HSynLPeu6Ei0wbm5uBAYGGtoNhUKhKFIIIS7l5To1RaZQKBQKvaAERqFQKBR6QQmMQqFQKPRCiV6DUSgKC2lpacTExJCSkmJoVxSKLCwsLHB2dsbU1DRf7ZXAKBSFgJiYGMqUKYObmxtCCEO7o1AgpeTmzZvExMRQpUqVfNlQU2QKRSEgJSUFe3t7JS6KQoMQAnt7+wKNqpXAKBSFBCUuisJGQf8mlcDkg/ikB0zdGMyt5FRDu6JQKBSFFiUw+eBgWDxLDkbSYuZuFh+IJDU909AuKRQKRaFDCUw+6OJbiX9GNsensi2fbwym3Zx97AiORUppaNcUinxx+/Zt5s2bl6+2c+bM4d69ezrzZenSpYwYMUJn9vLKnj176NSp03O3e15/o6Ki+O233567n7ySkJBAmzZt8PDwoE2bNty6dSvH6y5fvkzbtm2pUaMGNWvWJCoqSue+KIHJJ9UrlOGXQQ34eWB9hIAhvwTSf/ERzl+7a2jXFIrnpjAJTHHnaQKTnp5eYPvTp0/H39+f0NBQ/P39mT59eo7Xvfnmm4wZM4bz589z9OhRypcvX+C+H0eFKRcAIQR+L5fnFQ8HVhy+xJydoXT8bj996ldmdJvqlCtjbmgXFUWQzzacI/iqbr+o1KxozaSAWrmeHzduHOHh4fj6+tKmTRvKly/PH3/8wYMHD+jWrRufffYZycnJ9O7dm5iYGDIyMpg4cSKxsbFcvXoVPz8/HBwc2L17d472t2zZwscff0xGRgYODg7s3LmThIQEBg0aREREBJaWlixYsABvb+9H2q1evZrPPvsMY2NjbGxs2LdvH1FRUbzxxhskJycD8P3339OkSRP27NnDpEmTcHR0JCgoiO7du+Pl5cW3337L/fv3+fvvv6latSoDBw7EwsKCc+fOERsby+zZs58YuSQnJ/P+++9z5swZ0tPTmTx5Ml26dMn1+UVHR9O+fXsiIyPp168fkyZNYuLEiTg4ODBy5EgAJkyYgKOjI7/99hvnz5/H19eXAQMGULZsWTZt2kRKSgrJycns2rWLmTNnPvH8AZYvX853331HamoqDRs2ZN68eRgbGz/iy7p169izZw8AAwYMoGXLlsyYMeORa4KDg0lPT6dNmzYAWFlZ5XpvBUEJjA4wNTZiYNMqdK1die92hvHLv1FsOHWN9/yq8VZTNyxMjZ9pQ6EwJNOnT+fs2bMEBQWxbds21qxZw9GjR5FS0rlzZ/bt20dcXBwVK1Zk06ZNANy5cwcbGxtmz57N7t27cXBwyNF2XFwcQ4cOZd++fVSpUoWEhAQAJk2aRO3atfn777/ZtWsXb775JkFBQY+0nTJlClu3bqVSpUrcvn0bgPLly7N9+3YsLCwIDQ2lb9++WUlrT506xfnz57Gzs8Pd3Z0hQ4Zw9OhRvv32W+bOncucOXMAzShi7969hIeH4+fnR1hY2CP9Tps2jVatWrFkyRJu375NgwYNaN26NaVLl87xHo8ePcrZs2extLSkfv36dOzYkcGDB9O9e3dGjhxJZmYmK1eu5OjRo3h7ezNr1iw2btwIaKbY/v33X06fPo2dnR3btm0jNDT0iedfrlw5Vq1axcGDBzE1NeXdd99lxYoVvPnmmwwZMoThw4dTr149YmNjcXJyAsDJyYkbN2484W9ISAi2trZ0796dyMhIWrduzfTp058Qq4KiBEaH2Fqa8WlATV5v5MKXm88zY8sFfjt6ifEdatDBs4IKQ1XkiaeNNF4E27ZtY9u2bdSuXRuApKQkQkNDadasGR999BFjx46lU6dONGvWLE/2Dh8+TPPmzbM269nZ2QFw4MAB/vzzTwBatWrFzZs3uXPnziNtmzZtysCBA+nduzfdu3cHNFkPRowYQVBQEMbGxoSEhGRdX79+/awP16pVq9K2bVsAvLy8Hhld9e7dGyMjIzw8PHB3d+fChQtPPIP169cza9YsQLNP6fLly9SoUSPHe2zTpg329vYAdO/enQMHDjBq1Cjs7e05efIksbGx1K5dO+uanNo/fC65Pf/Tp09z/Phx6tevD8D9+/ezprUWLVqUo93cSE9PZ//+/Zw8eRIXFxf69OnD0qVLGTx48HPZeRZKYPRA1XJWLBpQnwOh8UzdFMy7K07QwM2OiZ1q4uVsY2j3FIqnIqVk/PjxvP3220+cO378OJs3b2b8+PG0bduWTz/9NE/2cvpylVNQzOPXzZ8/nyNHjrBp0yZ8fX0JCgpi7ty5ODo6curUKTIzM7GwsMi63tz8v2lpIyOjrPdGRkaPrG883s/j76WU/Pnnn1SvXv2Z9/c0e0OGDGHp0qVcv36dQYMG5do++8got+c/d+5cBgwYwJdffvlUXxwdHbl27RpOTk5cu3Ytx7UVZ2dnateujbu7OwBdu3bl8OHDOhcYtcivR17xcGDTB834opsX4XFJBHx/gP/9cYrYuyrflKJwUaZMGRITEwFo164dS5YsISkpCYArV65w48YNrl69iqWlJf379+ejjz7ixIkTT7TNicaNG7N3714iIyMBsqbImjdvzooVKwBNBJeDgwPW1taPtA0PD6dhw4ZMmTIFBwcHoqOjuXPnDk5OThgZGfHrr7+SkZHx3Pe7evVqMjMzCQ8PJyIi4gkhadeuHXPnzs0SwZMnTz7V3vbt20lISMha62natCkA3bp1Y8uWLRw7dox27doBz35euT1/f39/1qxZkzXllZCQwKVLT5Zl6dy5M8uWLQNg2bJlOa4d1a9fn1u3bhEXFwfArl27qFmz5lPvMT/oVWCEEO2FEBeFEGFCiHE5nB8ohIgTQgRpX0OynZshhDirffXJdnyE1p4UQjhkO95SCHEnm61nf7V6ARgbCfo1dGH3mJa83cKdDaeu0nLmHr7dEcr91Of/x1Ao9IG9vT1NmzbF09OT7du3069fPxo3boyXlxc9e/YkMTGRM2fO0KBBA3x9fZk2bRqffPIJAMOGDaNDhw74+fnlaLtcuXIsWLCA7t274+PjQ58+mn/nyZMnExgYiLe3N+PGjcv6UMzOmDFj8PLywtPTk+bNm+Pj48O7777LsmXLaNSoESEhIbmuizyN6tWr06JFCzp06MD8+fMfGQUBTJw4kbS0NLy9vfH09GTixIlPtffKK6/wxhtv4OvrS48ePahXrx4AZmZm+Pn50bt376z1DW9vb0xMTPDx8eGbb755wlbbtm1zfP41a9Zk6tSptG3bFm9vb9q0acO1a9cAzUjp4TrUuHHj2L59Ox4eHmzfvp1x4zQfvYGBgQwZovmINTY2ZtasWfj7++Pl5YWUkqFDhz73c3wmUkq9vABjIBxwB8yAU0DNx64ZCHyfQ9uOwHY0U3ilgUDAWnuuNuAGRAEO2dq0BDY+j49169aVL5pL8cnyneWB0nXsRtnoix1y7YkYmZGR+cL9UBQugoODDe1CiWHAgAFy9erVL6SvjIwM6ePjI0NCQl5If/ogp79NIFDm4TNWnyOYBkCYlDJCSpkKrARyj/N7lJrAXillupQyGY04tQeQUp6UUkbpw+EXgYu9JfNer8uqYY2wtzJj1Koguv14iOOXct4MpVAoiibBwcFUq1YNf39/PDw8DO2OQdDnIn8lIDrb+xigYQ7X9RBCNAdCgA+llNFoBGWSEGI2YAn4AcF56LOxEOIUcBX4SEp5riA38FQSIsDOPd/NG7rbs/69V/jzRAwzt16kx4+HCPCpyNj21XEua6lDRxWKF0fDhg158ODBI8d+/fVXvLy8DOTRkyxdujRf7bZu3crYsWMfOValShXWrl2b4/U1a9YkIiIiX30VF/QpMDnF5D4eNrIB+F1K+UAIMRxYBrSSUm4TQtQHDgFxwL/As7a4ngBcpZRJQohXgb+BJ742CCGGAcMAXFxcnud+/iPod1j3HgzeDs5182cDMDIS9KpXmVe9nPhpbzg/7Ytg27nrDG3mzjstq1LaXAX5KYoWR44cMbQLeqNdu3ZZC/WKvKHPKbIYoHK2985oRhZZSClvSikfft1ZCNTNdm6alNJXStkGjViFPq0zKeVdKWWS9ufNgGn2IIBs1y2QUtaTUtYrV65cfu4LXn4VyjjB2rch7X7+bGSjtLkJo9tWZ9dHLWnvWYHvd4fRctYe/giMJjNT5TdTKBRFE30KzDHAQwhRRQhhBrwGrM9+gRDCKdvbzsB57XFjIYS99mdvwBvY9rTOhBAVhDb4XAjRAM293dTRvTzCA1MLVjfsh7wZCjs+05ndSral+Pa12vz1bhOcy5bi/9acJuD7AxyO0MttKBQKhV7Rm8BIKdOBEcBWNMLxh5TynBBiihCis/ayD4QQ57TrJh+giSoDMAX2CyGCgQVAf609hBAfCCFi0IyITgshHm5h7Qmc1dr6DnhNG+2gczZHbGZK2EpWeLaFIz9C5D6d2q/jUpa/3mnCt6/5cis5ldcWHGb4r8e5dDNZp/0oFAqFPhF6+gwuEtSrV08+jB1/HqSUjNw9kv0x+/g5EXxTM+Cdg2Bh/ezGz0lKWgYL90Xw495w0jMkA5u6MaJVNawtTHXel8JwnD9/Ptc0JAqFIcnpb1MIcVxKWe9ZbdVO/nwghGDqK1NxsqrI/+ytuZl0FbZ+rJe+LEyNed/fg90ftaSLb0UW7o/Ab+Yelh++RHqGKnSm0A2FKV2/qgdTMPJSD2b37t34+vpmvSwsLPj777917osSmHxibWbN7JazuZN+j7EevmSc/BVCtuqtP0drC2b28mHDiFeoWt6KT/4+y6vf7WdfSJze+lSUHAqTwBR3CkM9GD8/P4KCgggKCmLXrl1YWlpmJQbVJSoOtgC8bPcyExpO4NNDnzLPuRrvr38f3j0MlnZ669Ozkg2rhjVi67nrfLH5Am8uOUqrl8vz8as1qFZePzUdFC+Yf8bB9TO6tVnBCzrkXHgKVD2YklYPJjtr1qyhQ4cOWFrqfv+dEpgC0s2jGydvnGRB2Fp8MpNpvmk09Fqq1z6FELT3dMLv5fIsPRjF97vCaD9nH/0buTLS34Oypc302r+i+KHqwZSsejDZWblyJaNHj37qNflFCYwO+Ljhx5xPOM94owj+uLieSmfWgFdPvfdrbmLM2y2q0qOuM99sD+GXf6NYe/IKI/09eKOxK6bGaga0SPKUkcaLQNWDKf71YB5y7do1zpw5o7cNpEpgdICFiQWzW8ymz8be/K+SK79s+h9mrk3B2unZjXWAg5U507p58UZjV6ZtOs+UjcEsP3yJCR1r0Orl8qrQmeK5kKoeTLGvB/OQP/74g27dumFqqp+oVPUVV0dUtq7M1Femcc4onRllTGH9+/CCQ8BfrmDNL4MasGRgPRAweFkgbyw+yoXruq3vrih+qHowJasezEN+//13+vbt+9R7KwhqBKNDWrm04i3Pt/j57M/4XjtIwIllUHfgC/VBCEGrlx1p5lGO5YcvMWdHKK9+u5/XGrgwus1LOFiZP9uIosSRvR5Mhw4dsuqRAFhZWbF8+XLCwsIYM2YMRkZGmJqa8uOPPwL/1YNxcnLKcZE/ez2YzMzMrDWUyZMn89Zbb+Ht7Y2lpWWu9WBCQ0ORUuLv759VD6ZHjx6sXr0aPz+/AtWDiY2NzbUezKhRo/D29kZKiZubW9aaSU48rAcTFhZGv379nqgHY2trm2M9mIEDB1K2bNlHbLVt25bz588/8fyz14PJzMzE1NSUH374AVdX10fWYMaNG0fv3r1ZvHgxLi4urF69GtDUg5k/f37WdFpUVBTR0dG0aNHiuZ9fXlEbLfOx0fJppGemM3TbUM7GHue32Nt4DN0HZd102sfzcPteKnN2hLL88CVKmRrzXqtqvNXUDXMT42c3Vrww1EbLF8fAgQPp1KkTPXvqf500MzOTOnXqsHr16iKbsl9ttCxEmBiZMLPFTKzMbRltb0XS38Mh03AbIm0tzZjcuRZbP2xOgyp2TP/nAm1m7+OfM9dynANXKBS6QdWDUSMYnY9gHhJ4PZAhWwfRKjmZr30/RDR58TuTc2J/aBxTN57nYmwiDarYMbFjTbycbQztVomnuIxgikI9mPzyvPVgigsFGcEogdGTwAD8fGYJs098w//dSuSN17dCubxFpOib9IxMVgVGM3tbCAn3UulRx5kx7arjaG3x7MYKvVBcBEZR/FBTZIWUgZ5v0apiU2bbWnHy70GQkWZolwAwMTbi9Yau7B7TkmHN3FkfdBW/WXv4bmco91OfPyJHoVAockIJjB4RQvB5i69wsrDnI6MEbu75wtAuPYK1hSnjX63B9tHNafFSOWZvD8H/6z2sC7qi1mcUCkWBUQKjZ6zNrPmm7QLumJgyNnQFGVeOG9qlJ3C1L82P/euyclgjypY2Y+TKILrNO8SJy09mYVUoFIq8oleBEUK0F0JcFEKECSHG5XB+oBAiTggRpH0NyXZuhhDirPbVJ9vxEVp7MntJZKHhO+2500KIOvq8t+ehul11JtQfy5FS5vyweQikpRjapRxp5G7PhhGvMLOnN1du36f7vEN88PtJrtwueFloReGmMGVTVun6C0Ze0vUD/N///R+1atWiRo0afPDBB3qZtdCbwAghjIEfgA5ATaCvEKJmDpeuklL6al+LtG07AnUAX6AhMEYI8XCL70GgNfD4FtYOgIf2NQz4Uce3VCC61Xyd7uUbstAsnX3/FI6IspwwMhL0qleZPR+15P1W1dh67jqtZu3h620XSX5Q8FTiisJJYRKY4k5hSNd/6NAhDh48yOnTpzl79izHjh1j7969Be77cfQ5gmkAhEkpI6SUqcBKIPecBY9SE9grpUyXUiYDp4D2AFLKk1LKqBzadAF+kRoOA7ZCiBeTDCyPjG/zPS8bWzE+/hBXLqw3tDtPpbS5Cf9rW51dH7WkXa0KzN0Vhv/XezkVfdvQrin0QPZ0/WPGjGHmzJnUr18fb29vJk2aBGhS2Hfs2BEfHx88PT1ZtWoV3333XVa6fj8/v1ztb9myhTp16uDj44O/vz+g+abdtWtXvL29adSoEadPn36i3erVq/H09MTHx4fmzZsDmg/oZs2aUadOHerUqcOhQ4cAzQikRYsW9O7dm5deeolx48axYsUKGjRogJeXF+Hh4YBmo+Xw4cNp1qwZL730Uo479JOTkxk0aBD169endu3arFu37qnP72G6/urVq2el1p84cSLffvtt1jUTJkzgu+++Y9y4cezfvx9fX1+++eYbli5dSq9evQgICMhKzpnT8wdNuv4GDRrg6+vL22+/nWOanHXr1jFgwABAk64/p0JiQghSUlJITU3lwYMHpKWl4ejo+NR7zA/6TBVTCYjO9j4GzWjkcXoIIZoDIcCHUspoNIIySQgxG7AE/IDgfPRXCbiWP/d1j4WJBbM7LKXPhp6MPjiBX1yaYm6Zc3bVwkIl21J817c2A5q48cHvJ+n907/M7u1LR+9Cpd3FihlHZ3Ah4cKzL3wOXrZ7mbENxuZ6XqXrL1np+hs3boyfnx9OTk5IKRkxYoRewuT1OYLJKYXv45N8GwA3KaU3sANYBiCl3AZsBg4BvwP/As8aO+alP4QQw4QQgUKIwLi4F18NsrJ9daZ5vUuwCcxY//oL7z+/1HUty7oRTfGsZMN7v53gu52hKtKsmJI9XXydOnW4cOECoaGheHl5sWPHDsaOHcv+/fuxscnbBt2npet/4403gGen61+4cGHWt/W0tDSGDh2Kl5cXvXr1Ijj4v++eD9P1m5ubP5GuPyoqKuu6vKTrnz59Or6+vrRs2TIrXX9uPEzXX6pUqax0/W5ublnp+h8+z+dN15/9+e/cuTMrXb+vry87d+4kIiIC0KTrf5j/LC+EhYVx/vx5YmJiuHLlCrt27WLfvn15bp9X9DmCiQEqZ3vvDFzNfoGU8ma2twuBGdnOTQOmAQghfgNCC9qf1u4CYAFoNlo+6yb0gV/ddxgUuY0lyWHUPvQlAU3GG8KN58bBypzfhjZk/J9nmL09hLAbSXzV0xsLU5XXTJc8baTxIlDp+ot/uv61a9fSqFEjrKw0VXA7dOiQ9UVAl+hzBHMM8BBCVBFCmAGvAY8sPDy2RtIZOK89biyEsNf+7A14A9ue0d964E1tNFkj4I6UstBMjz3O+wG/Uj/diCkhKwi5pr9sArrG3MSYr3v7MKZdddafusprCw5zI7FwRsUp8o5K11+y0vW7uLiwd+9e0tPTSUtLY+/evUVrikxKmQ6MALaiEY4/pJTnhBBThBCdtZd9IIQ4J4Q4BXwADNQeNwX2CyGC0Yw2+mvtIYT4QAgRg2aEcloI8bCU22YgAghDMxp6V1/3pgtMzK34qtUcymRkMnr7OySlJhnapTwjhOA9v2rM71+Hi9cT6fr9QYKvqpozRZns6fq3b9+ela7fy8uLnj17kpiYyJkzZ7IWmKdNm8Ynn3wC/JeuP7dF/uzp+n18fOjTR7PrYPLkyQQGBuLt7c24ceNyTdfv5eWFp6cnzZs3z0rXv2zZMho1akRISEiB0vV36NAh13T9aWlpeHt74+npycSJE59q72G6fl9fX3r06PFEuv7evXvnmK7/m2++ecJW27Ztc3z+2dP1e3t706ZNG65d03yHHjJkSNY61Lhx49i+fTseHh5s376dceM0O0QCAwMZMkSzE6Rnz55UrVoVLy8vfHx88PHxISAg4Lmf47NQucjymYssMzUVIzOzAvtwfMtoBl/fRis7T74O+L3IVZ88e+UOQ5YFcjcljW9fq02bmrqPRCkJqFxkLw6Vrv/5ULnIXjCJu3cT7t+atKtPLPE8N3XbzGBUuiXbb53jl5P524dgSDwr2bBuRFOqlbdi2K+B/LQ3XC3+KxSodP2gRjD5GsGkxlwhIiAAywb1qTx/foFHHTL2PB/+1Zk9lhYsabeUOhXqFsieIbifmsFHa06x6fQ1etV1Zlo3L8xM1PeXvFJcRjAqXX/xQ6XrzycFmSJL+OUXYr/4koozZ2IT8PzpJR4ncf/XvHZxIfdLleWP7htwKJXznoLCTGam5NudoXy7M5QGbnbMf6MudqULPo1YEiguAqMofqgpMgNQ9vXXsfDxJnbaNNK1UTEFoUzTD5lt7MzdtETG7hpJembRS8tiZCT4sM1LfNe3NkExt+n6w0FCY3OPllEoFMUbJTD5RBgbU3HqVDKSk4n94ulx6XnCyIjqXRfxye1kjsaf5oeT3xfcpoHo7FORVcMacS81g+7zDrHn4pM7iRUKRfFHCUwBMPfwwOHtt7m7cSOJe/YU3GBZN7o2m0SPxCQWnV3Mnmgd2DQQtV00O/+d7SwZtPQYSw9GqsV/haKEoQSmgDgMG4q5RzWuT/6MjCQd7GWpM4DxNrWpkZrGx/vGEZ0Y/ew2hZRKtqVYM7wx/jUcmbwhmInrzpKWkWlotxQKxQtCCUwBEWZmOH3+OemxscTNnq0DgwLzLj/w9e0USLvH//b8jwcZD57drpBS2tyEn/rX5e0W7iw/fJmBPx/lzr3CUTpa8R+FKV2/qgdTMPJaD2bs2LF4enpmZcbWB0pgdEApX1/s3nyDW7/9zr3jOqhYae1E5faz+CL2BucTzvPlER2s8RgQIyPB+A41mNnTm6ORCXSbd5DI+GRDu6XIRmESmOJOYagHs2nTJk6cOEFQUBBHjhxh5syZ3L2r+2wcSmB0RLmRIzGtVIlrn0wk84EORhyePWhZpR2D7yTxZ+ifrAt7ej2KokCvepVZMaQRt+6l0vWHgxwKjze0Swotqh7MoxT3ejDBwcG0aNECExMTSpcujY+PD1u2bHnqPeYLKWWJfdWtW1fqksT9B2Rw9Zdl7OxvdGMwKV6mfVVNDlrsI+v+WldeuHlBN3YNzKX4ZNn66z2y6vhNcsXhS4Z2p1AQHByc9fO1adNkVP83dPq6Nm3aU/uPjIyUtWrVklJKuXXrVjl06FCZmZkpMzIyZMeOHeXevXvlmjVr5JAhQ7La3L59W0oppaurq4yLi8vV9o0bN6Szs7OMiIiQUkp58+ZNKaWUI0aMkJMnT5ZSSrlz507p4+MjpZTy559/lu+9956UUkpPT08ZExMjpZTy1q1bUkopk5OT5f3796WUUoaEhMiH/8e7d++WNjY28urVqzIlJUVWrFhRfvrpp1JKKefMmSNHjhwppZRywIABsl27djIjI0OGhITISpUqyfv378vdu3fLjh07SimlHD9+vPz111+z+vXw8JBJSUk53t/PP/8sK1SoIOPj4+W9e/dkrVq15LFjx2RkZKSsXbu2lFLKjIwM6e7uLuPj4x/p52H7SpUqZT2X3J5/cHCw7NSpk0xNTZVSSvnOO+/IZcuWSSmlHDx4sDx27JiUUkobG5tH/LO1tX3C561bt8omTZrI5ORkGRcXJ6tUqSJnzZqV4/1l/9t8CBAo8/AZq0YwOsTqlabYdO3KzcWLSbmgg4JRpe0x6TyXGTGXsJaC0XtGk5ha9PeVuNhb8ue7TWhazYGP155hyoZgMjJVhFlhQdWDKf71YNq2bcurr75KkyZN6Nu3L40bN8bERPfVW/RZD6ZE4jhuLEn793Ntwie4rVqJKOgvrXp7HHz6MfP8GgZXTOXTg58yu+XsIpcU83GsLUxZPKAe0zafZ8nBSCLjk/iub23KWJga2jWDU+Hjjw3av1T1YIp9PRjQTNlNmDABgH79+uklX5oawegYY1tbKkz8hJRz50iqNUZrAAAgAElEQVTIIf14vmj3JXXNy/PhPdhxeQe/BP+iG7sGxsTYiEkBtZjWzZN9ofH0+PEQ0QlqsdgQqHowJaseTEZGBjdvauo9nj59mtOnT2eN9nSJGsHogTLt2mHl70/cd3Mp07o1Zq6uBTNoYQ1df+DNZQEEVa/PN8e/wdPBk7qORS8pZk683tAVN/vSvLP8OF1+OMiCN+pSz83O0G6VKLLXg+nQoUNWPRIAKysrli9fTlhYGGPGjMHIyAhTU1N+/PFH4L96ME5OTuzevfsJ29nrwWRmZlK+fHm2b9/O5MmTeeutt/D29sbS0jLXejChoZry3P7+/ln1YHr06MHq1avx8/MrUD2Y2NjYXOvBjBo1Cm9vb6SUuLm55RgM8JCH9WDCwsLo16/fE/VgbG1tc6wHM3DgQMqWLfuIrbZt23L+/Pknnn/2ejCZmZmYmpryww8/4OrqypAhQxg+fDj16tVj3Lhx9O7dm8WLF+Pi4sLq1asBTT2Y+fPns2jRItLS0mjWrBkA1tbWLF++XC9TZCrZZT6TXT6LtNgbRHTqhEWNGrgsW6qbKa1/xpF4dD6vvVyH+wL+CPijSCbFzI3wuCSGLAvkyq37TO/hRfc6zoZ26YWhkl2+OFQ9mOej0Ca7FEK0F0JcFEKECSHG5XB+oBAiTggRpH0NyXZuhhDirPbVJ9vxKkKII0KIUCHEKm055qfaMgSmjuUpP+Yj7h09ym3tN4gC03oSZeyqMft6LImpd/m/ff9XJJNi5kbVclasfbcJ9dzKMvqPU3y15QKZavFfUURR9WD0OIIRQhgDIUAbIAY4BvSVUgZnu2YgUE9KOeKxth2BUUAHwBzYC7SSUt4VQvwB/CWlXCmEmA+cklL+mJutp6HPEQxoFusuDxhIyvnzuG/ciKljzottz0XMcVjcmnU1WvHJ/RAGew5mVN1RBbdbiEjLyOTTdef4/ehl2tVy5Js+vliaFe/Z3OIyglH1YIofBRnB6PO/tgEQJqWM0Dq0EugCBD+1lYaawF4pZTqQLoQ4BbQXQqwGWgH9tNctAyYDP+rYd50ghMDp8ylEdO7C9SlTcP5+bsGnypzrwiuj6bJ/FkENe7L47GJ8yvng55L7JreihqmxEV9088SjvBVTNwXT88d/WTywHk42pQztmuIZHDlyxNAu6I127dplLdQr8oY+p8gqAdkzNcZojz1ODyHEaSHEGiFEZe2xU0AHIYSlEMIB8AMqA/bAba3w5GQzJ1sGxczVlXIfvE/Szp0kbt2qG6MtxkIFL8ad3UsNWw8mHJhQpJNi5oQQgkGvVGHxwPpcTrhH5+8PEhR929Bu6ZWSvB6qKJwU9G9SnwKT01f1x73dALhJKb2BHWhGJEgptwGbgUPA78C/QPozbOZo6wmnhBgmhAgUQgTGxcU93x3lE7sBA7CoVYvrn08l47YOPiRNzKDbT5in3GH2PROEEEU+KWZu+FUvz1/vNsHcxIg+P/3LhlNXDe2SXrCwsODmzZtKZBSFBiklN2/efCLC7nnQ5xpMY2CylLKd9v14AClljruEtGs2CVLKJ7YHCyF+A5YD/wBxQAUpZfrjfeTFVnb0vQaTnZQLF4js2QubgAAqfvmFbowe+AZ2TGaf///xXsRKenj0YHKTybqxXci4mfSA4cuPcyzqFqNaezDS36PIbzbNTlpaGjExMaSkpBjaFYUiCwsLC5ydnTE1fXQDdGFYgzkGeAghqgBXgNf4b+0EACGEk5TymvZtZ+C89rgxYCulvCmE8Aa8gW1SSimE2A30BFYCA4B1T7NVWLB4+WXsBw/m5k8/Yd2pI1bajVgFoskHcPEfmh/8iaEt3mZhyEp8y/vStVrXgtsuZNhbmbN8SEM+/ussc3aEEh6XzMye3liYGhvaNZ1gamqalUpFoSgu6HUfjBDiVWAOYAwskVJOE0JMQZMobb0Q4ks0YpAOJADvSCkvCCEsgBNaM3eB4VLKIK1NdzTiYgecBPpLKR/kZutp/r3IEQxA5oMHRHbthkxNxX39OozysUHsCW6Gw/xXyKjciLcd7QiKO8WKV1dQ3S5vKS6KGlJK5u+N4KutF/B2tmXhG3Upb53/IbxCoXh+8jqCURstX6DAANwLDORS/zewG/AmjuPH68bo0YWw+SNutptK7+i/sTCxYGWnlZQxK6Mb+4WQreeuM2plELaWpiwaUI9aFfOWeFGhUBScQrHRUvEklvXqUbZfXxJ++ZX7QUG6MVp/CLj7Yb/rS2bVGc3VpKt8cuCTYr1g3K5WBda8o0ml0Wv+v2w7d93AHikUisdRAmMAyo0ejYmjI9cmTkSmphbcoBDQ5QcwMqH2vrl8WGcku6J3seycjpJtFlJqVbRh3XtN8XAsw9vLj/PjnvBiLaoKRVFDCYwBMLayosLkSTwIDSN+wULdGLWpBK9+BZf/5Y3bd2jj2oY5J+YQeP3FTgG+aMpbW7BqWCM6ejkxY8sFPlp9mgfpz59dV6FQ6B4lMAaiTMuWWHfqRPxPP/EgNFQ3Rr37wMudELunMsXjdSqXqcyYfWOIv1+8SxNbmBozt29tRrX24M8TMfRfdISbScVvT5BCUdRQAmNAHD8ej3Hp0lz95BNkPmpaPIEQ0GkOmFtjtfFDvm42g6TUJMbsHVOskmLmhBCCUa1fYm7f2pyOuUPXeQcJiS361T8ViqKMEhgDYmJnh+OEj0k5dZpb2sJLBcaqHATMgWuneOnM33za+FMCYwOZe3KubuwXcgJ8KrLq7cakpGXSfd4hdl+8YWiXFIoSixIYA2PdqROlWzTnxjdzSI25ohujNQLA+zXYN4sAi4r0eqkXS84uYfflJ4tBFUd8K9uy7r2muNhZMnjpMZYciFSL/wqFAVACY2CEEDhNmoQQguuffqq7D8IOM6BMBVg7nLG1P6CmfU1NUsy7xSspZm5UtC3Fmnca07qGI1M2BjPh77OkZWQa2i2FokShBKYQYFqxIuX+N5rkQ4e48/c63RgtZQtdvof4EMz3fMXslrMRQvDhng9JSS8Z+a4szUyY378u77Ssym9HLjNgyVFu39NBWLhCocgTSmAKCWX79qVUnTrETp9OeryOor6qttJswjw8j0rxkXzZ7Esu3rrIF0d0lGyzCGBkJBjb/mW+7uVDYNQtus07RERckqHdUihKBEpgCgnCyAinqZ8j793j+tRpujPcZgrYVYG/36F5udoM9RrK2rC1rA0t3lX4HqdHXWdWDG3InftpdP3hIAfDinfotkJRGFACU4gwd3fH4b13SdyyhcSdO3Vj1Kw0dJ0Pd2Jg68e85/seDZ0aMu3INC4kPDUXaLGjvpsd695rSgUbC95ccpQVRy4Z2iWFolijBKaQYT94MObVq3P9sylkJOpoH4dLQ2jyPpz4BeOwncxoNgMbcxs+3P0hd1Pv6qaPIkJlO0v+fKcJzT0cmLD2LJ9tOEe6WvxXKPTCMwVGCFFVCGGu/bmlEOIDIYSt/l0rmQhTU5ymTiU9Pp4bM2fpzrDfBChfE9a/j70UfN3ia64nXy/2STFzooyFKYsG1GdQ0yr8fDCKIb8EcjclzdBuKRTFjryMYP4EMoQQ1YDFQBXgN716VcIp5eWJ3cCB3P7jD5KPHNWNURNz6DYf7sXD5o/wLe/L6Hqj2R29m5/P/aybPooQxkaCTwNq8kU3Lw6ExtNj3iEu37xnaLcUimJFXgQmU0qZDnQD5kgpPwSc8mJcCNFeCHFRCBEmhBiXw/mBQog4IUSQ9jUk27kZQoiz2lefbMerCCGOCCFChRCrhBBm2uPm2vdh2vNuefGxsFLu/RGYurhw7dOJZOqqjK6TD7QYB2f/hLN/0b9Gf9q6tuXbE99y7Pox3fRRxOjX0IVfBjXgRuIDus47yNHIBEO7pFAUG/IiMGlCiL5oyhNv1B4zfcr1QFbZ4x+ADkBNoK8QomYOl66SUvpqX4u0bTsCdQBfoCEwRghhrb1+BvCNlNIDuAUM1h4fDNySUlYDvtFeV2QxKlUKpylTSLt0mfjvv9ed4Vc+hEp1YdNoRFIsnzX5DJcyLozZO4a4e3G666cI0aSaA3+/1xTbUqa8vugwa47HGNolhaJYkBeBeQtoDEyTUkYKIaoAy/PQrgEQJqWMkFKmoilz3CWPftUE9kop06WUycApoL0QQgCtgDXa65YBDwvQd9G+R3veX3t9kaV0o4bY9urJzSU/c//sOd0YNTbRRJWl3Yf1H2BlWprZLWdzL/0eY/YV/6SYuVHFoTRr321Kgyp2fLT6FNP/uUBmZslam1IodM0zBUZKGSyl/EBK+bsQoixQRko5PQ+2KwHZ85LEaI89Tg8hxGkhxBohRGXtsVNAByGEpRDCAfADKgP2wG3tlN3jNrP6056/o72+SFN+zBhM7O259sknyDQdLUSXewlaT4bQrXDyVzzKejCx0USOxx7nuxPf6aaPIoiNpSlL32rA6w1dmL83nLeXHyf5QckUXIVCF+QlimyPEMJaCGGH5oP/ZyHE7DzYzmn08PhXwg2Am5TSG9iBdgQipdwGbAYOAb8D/wLpz7CZl/4QQgwTQgQKIQLj4gr/lJCxtTUVJn3KgwsXuLlEh4vxDd4Gt2awZTzcukRA1QB6v9Sbn8/9zM7LOtqDUwQxNTZialdPJgfUZOf5WFrP3suY1af441g0EXFJJS7iTqEoCOJZ/zBCiJNSytraBfjKUspJQojTWlF4WrvGwGQpZTvt+/EAUsovc7neGEiQUtrkcO43NNNy/wBxQAUpZXr2PoQQW7U//yuEMAGuA+XkU26wXr16MjCwaFR8jBk5iqTdu6ny99+Yu1fRjdHbl2FeE83i/4ANpMp03vznTS7dvcSqTqtwsXbRTT9FlAOh8Sw9FEXgpQRu39OMHu1Lm1HPrSz13eyo52ZHrYrWmBqr7WSKkoUQ4riUst4zr8uDwJwB2qIZXUyQUh7Lo8CYACGAP3AFOAb0k1Key3aNk5TymvbnbsBYKWUjrdjYSilvCiG80YRF+2pFZTXwp5RypRBiPnBaSjlPCPEe4CWlHC6EeA3oLqXs/TQfi5LApMfFEd4pAPNq1XD99ReEkY4+1E78CutHQLsvofG7XEm6Qu8NvXEq7cTyV5djYWKhm36KMJmZkoj4JI5F3eJYVAKBUbe4nKAJaS5laoxvZVvqu5WlnpsdtV1sKWPxzBgYhaJIo0uB6QVMBA5KKd8RQrgDM6WUPfLgxKvAHMAYWCKlnCaEmAIESinXCyG+BDqjmf5KAN6RUl4QQlgAJ7Rm7gLDpZRBWpvuaAIG7ICTQH8p5QNtm1+B2lpbr0kpI57mX1ESGIDbf63l2scfU2HSp5Tt21c3RqWE3/pA5F54ez+Ue4l9Mft4b+d7dK3Wlc+bfq6bfooZsXdTCHwoOJcSCL56l0wJRgJqOFlrRziakY6jtRJpRfFCZwJTnClqAiOlJHrwEO4HBeG+aSOmTnnajvRsEq/DvEZg5w6DtoGxCXNPzmXB6QV81uQzunt0100/xZikB+mcvHyLY1G3CIxK4OTl29xP05TBrmxXivqumim1+m5lqVrOCiOjIh3gqCjh6HIE4wzMBZqiWTQ/AIyUUhb5zQJFTWAAUmNiiAjoTOkGDXCe/yM6i8Q++xeseQv8PoEWY8jIzGD4juGciD3B8leXU8O+hm76KSGkZWQSfPVu1pRa4KUE4pM0tWhsLU2p5/rfOo5nJWvMTYwN7LFCkXd0KTDb0ayB/Ko91B94XUrZpsBeGpiiKDAACcuWEfvldCrOmoVNp466M7xmEASvg6G7wMmHhJQEem3ohZmRGasCVmFtZv1sG4ockVISdfOeVnA0ohMRnwyAuYkRPtnWceq4lMWmlFrHURRedCkwQVJK32cdK4oUVYGRGRlE9etH2uVo3DdvwqRsWd0YvpcA8xqDpR0M2wMm5gTdCOKtLW/xivMrfOv3LUZCRUzpivikB5rRTVQCxy7d4tyVO6RnSoSA6o5lHlnHqWhbytDuKhRZ6FJgdgBL0exHAegLvCWl9C+ok4amqAoMQEpICJE9emLdvj2VZn6lO8Mh2+C3XtB0pKZYGbA8eDkzjs3g/drvM9RrqO6m5RSPcC81naDo21nBAycu3SI5VbOOU8m2FPW0I5z6bmV5qXwZtY6jMBi6FBgX4Hs06WIkms2PH0gpL+vCUUNSlAUGIO67ucTPm0flBT9h1by57gyv/wBO/AKDtoBLI6SUjNk3hq1RW3G1diXAPYCAqgFUtKqouz4VT5CekcmF64lZI5xjkQncSHwAQBkLE+q5PhQcO7ydbbAwVes4iheDXqPIhBCjpJRz8uVZIaKoC0xmaiqR3buTmXwP9w0bMLYqrRvDDxLhx6YgjGD4ATC3Ii0jjY0RG1kfvp7AWM0zq1+hPgHuAbR1a0tpUx31rcgVKSUxt+5zLCohK1ot9EYSAGbGRng522im1FztqOtalrKlzQzssaK4om+BuSylLPLbvIu6wADcO3mSS/1ep2zfvlT4dKLuDEcdgKWdoN4g6PRoZqArSVfYGK4Rm8uJl7EwtsDf1Z/OVTvTsEJDjI3UN+kXxa3kVI5fusWxS5rAgdMxt0nL0PxPe5S3yppSq+9mh3PZUmp6U6ET9C0w0VLKys++snBTHAQG4Pq0L7i1fDmuK5ZjWaeO7gxv+RgO/wD9/4JqTy65SSk5FXeKDeEb+CfqHxJTEylfqjwdq3akS9UuVLWtqjtfFHkiJS2D0zF3/otWu3SLxBRNwk5Ha3ON4Gin1mo4WWOs1nEU+UCNYPJAcRGYzORkIgI6IywsqLL2L4zMzXVjOO0+/NRCM2X27r9QKvdK2Q8yHrA3ei/rw9dz4MoBMmQGNe1r0rlqZzpU6YCdhZ1ufFI8F5mZkpAbiVlTaoFRt7hy+z4AVuYm1HaxzYpWq125LKXM1OhT8WwKLDBCiERyyEaMJmtxKSmlScFcNDzFRWAAkg4cJHrIEOyHv035UaN0Z/jKCVjUGrx6Qfef8tTk5v2b/BP5D+vD13M+4TwmwoRXnF+hS9UuNHdujpmxWhswJFdu388Sm2NRCVyMTURKMDES1Kpkg1/1cvRv5IqDlY6+qCiKHSpVTB4oTgIDcHXsOO5s2kSVP9dgUb267gzv/gL2zoA+y6FGwHM1DbkVwobwDWyM2Ej8/XhszG1o79aezlU74+XgpdYECgF37qdx4rJmhHM0UhNAYGZiRI86zgxpVoWq5awM7aKikKEEJg8UN4FJv3WLiE4BmDo54bbyd4SJjgaZGWmwyB9uRWkKldV+U1MZ83l8y0znyLUjrAtfx67Lu3iQ8QA3azc6V+1MJ/dOOFnpKK+aosCExyWxaH8kf56IITU9k9Y1HBnW3J36bmXVFwIFoAQmTxQ3gQG4+88/XPlwNOXHjMF+8CDdGU6IgL/fhcv/gsNL0PozqN4B8vGBk5SaxLZL21gfvp7jsccRCOpXqE/nqp1p7dpahTwXEuKTHvDLv5f49d8obt1Lw6eyLcOaudOuliMmqgZOiUYJTB4ojgIjpSTmvREkHzqE+/p1mLnoMBZDSriwCXZMgpth4NoU2n4Olerm22RMYgwbIjawIXwD0YnRlDIphb+LJuS5QYUGKuS5EHA/NYM/T8SwaH8EUTfvUdmuFIObVqFXvcqUNi/yS7GKfKAEJg8UR4EBSIuNJaJjJyw8PXH5eYnupzUy0uDEMtgzHZLjoFZ38P8U7PJfafNhyPO68HVsjdxKYloijpaOdHLvROeqnXG3ddfhDSjyQ0amZMf5WBbuiyDw0i1sSpnyekMXBjZxo7yqeVOi0GWqmJyiye4AgcD/nlXUqzBTXAUG4NaqP7g+aRJOUz/HtmdP/XTyIBEOfguHvofMdGgwDJp/pEmWWRCzGQ/YHb2bDeEbOHjlIBkyg1r2tbJCnsta6Ci5pyLfHL90i0X7I9hy7jqmRkZ08a3I0ObuvORYxtCuKV4AuhSYz4CraFL2C+A1oAJwEU0FypZPadse+BZNRctFUsrpj50fCMxEU1IZ4Hsp5SLtua+AjoARsB1NDRophOgDTNDa3CSl/L9n2cqN4iwwMjOTywMGknLhAu4bN2LqWF5/nd29qok0C1oB5mWg2f+gwdtgWvBvtfH349kcsZkNERu4kHABE2FCM+dmWSHPpsYqrb0huXQzmcUHIvkjMJqUtExaVi/HsGbuNK5qrwICijG6FJgjUsqGjx07LKVsJIQ4JaX0yaWdMRACtAFigGNAXyllcLZrBgL1pJQjHmvbBI1YPMzgeAAYD5xBUya5rpQyTgixDPhFSrkzN1tPozgLDEBqVBQRXbpi1bwZznPn6r/D2HOwfRKEbQcbF/CfCJ49wUg3C8IXEy6yIXwDmyI3ZYU8d3DrQOeqnfF08FQfaAbkVnIqyw9fYtm/UcQnpVKrojXDmrvzqpcTpiogoNiRV4HJy28+UwjRWwhhpH31znbuaerUAAiTUkZIKVOBlUCXPPT30K4FYAaYA6ZALOAOhEgp47TX7QB65NFmicPMzY1y748gcfsO7m7dpv8OHWtB/zXw5jrNrv+/hsLClhCxVyfmq9tV56P6H7G953bm+c+jiVMT1oatpd/mfnRZ14VFZxZxPfm6TvpSPB9lS5vxvr8HB8a2Ynp3L1LSMhi5MogWX+1m0f4IElPSDO2iwgDkZQTjjmaaq7H20L/Ah2imoupKKQ/k0q4n0F5KOUT7/g2gYfYRhnbU8SUQh2a086GUMlp7bhYwBM203PdSyglCiLJoRjGvoBkVrQLMpJQBT7OVG8V9BAMg09OJ6t2HtBs3qLpxA8a2uad70SmZmXBmNez6HO5EQ7U2mvoyjjV12k1iaiLbojQhzydunEAgaODUQBPy7NIaS1NLnfanyBuZmZI9ITdYsC+CwxEJlDE3oa82IEAVTyv6GDyKTAjRC2j3mMA0kFK+n+0aeyBJSvlACDEc6C2lbCWEqIZG1PpoL90OjJVS7hNCBACfAJloatO4Sym75WYrB7+GAcMAXFxc6l66dEkv91+YSDl/nsievbDp0oWKX0x7sZ2npcDRn2Df15CaCL6vg98EsNb9xsrou9FZJQVikmIoZVKKNq5tCKgaQH3H+irk2UCcjrnNwv2RbD5zDQEE+FRkSLMq1KpoY2jXFPlEl2swzsBcoCmaqasDaBbcY57RrjEwWUrZTvt+PICU8stcrjcGEqSUNkKIMYCFlPJz7blPgRQp5VePtRkGVHu40J+Traf5WBJGMA+5Mfsbbi5YgMuSxZRu0uTFO3AvAfbNhKMLwdgUGo+Aph9oggJ0jJSSkzdOsj58PVujtpKUloSjpSMBVTWF0txtVMizIYhOuMfPB6NYeewy91IzeKWaA0Obu9Pcw0GtnxUxdCkw29FEkP2qPdQfeF1K2eYZ7UzQTFX5o5lOOwb0k1Key3aNk5TymvbnbmhGKY20kWJDgfZopsi2AHOklBuEEOWllDe002W70YxUQnKz9TQfS5LAZD54QGSXrsi0NNw3rMfI0kBTRwmRsHMKnPsLSpeDluOgzgCN6OiBlPQU9kTvYV34Og5dPUSmzMTLwYuAqgF0cOuArcULmjJUZHHnXhq/Hb3MzwcjuZH4gJcrlGFIM3c6+1TEzEQFBBQFdCkwQVJK32cdy6Xtq8AcNCHFS6SU04QQU4BAKeV6IcSXQGcgHUhAE/Z8QTsCmYcmikwCW6SUo7U2fwceRq5NkVKu1B7P0dbT/CtJAgNw79gxLr3xJnYDBuA4fpxhnYk5Dts+gcuHwN5Dk+Ps5Y75Sj2TV+Lvx7MpYhPrw9cTcisEEyMTWji3IKBqAM0rqZDnF01qeibrT11l4b4ILsYm4mhtzsAmVejX0AWbUup3UZjRpcDsAJYCv2sP9QXeklI+WYGqiFHSBAbg2mefcXvVH7j9/hulfHKMMH9xSAkX/9GknokPAZfG0HYqOD/z77bAXEy4yPrw9WyK2MTNlJvYmtvSoYom5LmWfS01ZfMCkVKyLzSeRfsj2B8aT2kzY/rUd+Gtpm5UtlNBGoURXQqMC/A9migyiWZh/QMp5WVdOGpISqLAZCQlEdEpAOMyZajy5xqEWSGozZKRrk0986U29Uw3beoZ/a+VpGemc+jqIdaHr2f35d2kZqbibuNOQNUAOrl3okLpCnr3QfEfwVfvsmh/BOtPXUUCHTwrMKy5O97OaiqzMKHvipajpJRz8uVZIaIkCgxA4u7dxLzzLg7vj6Dce+8Z2p3/eJAIh+ZqXhlpUH8INB8Dpe1fSPd3U+9mhTyfvHESgaC1a2umNp2qwp1fMNfu3GfpwSh+O3KZxAfpNKxix7Dm7vhVL4+RKvNscFTJ5DxQUgUG4Mr/PuLutm24r/0L82rVDO3Oo9y9Bnu+gJPLwawMNPsQGg4H0xe3fyL6bjRrw9ay+OxivBy8mNd6HtZm1i+sf4WGxJQ0Vh2LZsmBSK7eSaFqudIMbeZO19qVsDBVYeeGQt8CEy2lrJwvzwoRJVlg0hMSiHi1I2aurrj+tgJhXAj/WW+c16SeCd0K1s6a1DNevXWWeiYv7Li0gzH7xuBh68H8NvOxsyhYIk9F/kjLyGTzmWss2BfBuat3cbAyY0BjN/o3cqVs6UIwzVvCUCOYPFCSBQbgzvr1XP2/sTh+/DF2b75haHdyJ3KfJuLs2imo4AVtPoeqfi+s+wNXDjBq9ygqWVViYduFlLfUY+JQxVORUvJv+E0W7o9g98U4LEyN6F2vMoNfqYKrvSpU96IosMDkkqYfNPtSSkkpi3yloZIuMFJKot9+m3uBx6m6YT2mlSoZ2qXcycyEs39q9tDcuQzVWmuqalbwfCHdH7t+jBE7R2BnYceidouoZFWIn1UJISQ2kUX7I/j75FXSMjNpX6sCQ5q5U9dVlXPQNwZPFVMUKOkCA5B29SoRnQIoVacOlRcuKPzhuWkpcGyhJitAyl1N6plWE8C6ot67Ph13muNHnYEAACAASURBVOE7hmNpYsnCtgupYpP/AmsK3XHjbgrL/o1i+eHL3LmfRl3Xsgxt5k6bmo4Yq4AAvaAEJg8ogdGQsHwFsVOn4jT9S2y7djW0O3njXgLs/xqOLgBhDI3fhaajwEK/C/EXEy4ybPswABa0WUB1u+p67U+Rd5IfpLM6MJrFByOJTriPm70lg5u507OOM6XMCuEaYxFGCUweUAKjQWZmcun1/qRGROC+eRMm9i8mLFgn3IqCnZ/D2TX/396Zh1dVXf3/s+69uZkT5hBlSIITaJ1AxBGLldehxddXrUP1JyqgOFu1TogoWBV9raI4QW2tr2/V+tYKtaI4IKBYAQcQcYAENBCGMGWe7l2/P85JchNCuEBObob1eZ7z3H3O2XuftQm536x19l4bkno4qWcGj/Ys9QxA3o48xrw7hoqaCp79xbP8rOfPPHuWsefUhMK8s2Ijz89fzVf5O+iaFMelx2Xx/47rT4+U+Fib1yEwgYkCE5h6KlevJu8/zyH1tF+w/2OPxdqcPWfdUnh3IqxdCN0GOKlnBv7Ks9Qz+cX5jH13LFsrtjL91OkM6e199gFjz1BVFq/ZxvPzc3lv5UaCAR/nHt2HMSdlM6BnSqzNa9eYwESBCUxDCp95hs1PTKPP09NJHbHTTgdtH1X4/h2YOxEKv4O+w2DkZOg71JPHbSzdyNi5YykoKeDxnz/OCfuf4MlzjH1n9eYS/rgwj9eX5lNVE+YXAzMYd3IOx2R1bfvvHV1UlVBYqQ4pVaEwNaEw1SGlOhR2j4blmlCYql2Uq0NhBu2XxuD+ezft3gQmCkxgGqJVVeSddz6h7dvJeeuf+FNbPpV+qxCqgS9egg9/D6WbYOAox6PpPqDFH7W1YitXzb2KVdtX8ejJj3Jq/3afoq9DU1hSyUuL1vKXRWvYVlbNEX27MPakbIZmd6MmpNS4X97VofBO5Wr3S7qpcuQXvPNl3nS5sTjsSih21a4luWp4DneeMXCv2prARIEJzM6UL1/OmgsupMv555N536RYm7NvVJbAoqfg42kQqoQhV8Lw21s89UxRVRHj3xvPisIVTD5hMr8a8KsW7d9oecqrQvzf5/nMXJDLmi1lLd5/MOAjzifEBXwEfD6C/tqyEOf3EdxFOa62XaNywO/24Zbj/OK227nstGtUDohrR305Od5PUnDvVpuYwESBCUzTbHx4Klv/9Cf6/eVFkod6E15qVYo3OIk0P/8LBFPgxJtg2DUtmnqmrLqM6z+4nsUbFnPPcfdw/kHnt1jfhneEwspH329i3faKXXyxi/ulvBtxCDhf3gGf4PdJuwm77S0mMFFgAtM04fJyckedDT4h58038SUkxNqklmHzd07qme/fhrT9YcQEOPwCaKGtlCtqKrjlo1uYnz+fW4fcymWHXtYi/RpGWyNagfE0qZOInC4i34nIKhHZaYcrERktIptF5Ev3GBNxb6qIrBCRlSIyTdw/CUTkAhFZ5t6bGlE/XkRedZ/1bxHJ8nJsHRlfYiKZk++neu2PFE6fHmtzWo6eB8PFr8DotyClF/xjPDw3HFa93yLdJwQSePyUxxnZfySPLnmUZ756hs78B5xheCYw7q6U04EzgEHARSIyqImqr6rqke4x0217PHACcDhwGHAMMFxEugOPAKeq6qFAhojUvlW9EtimqgcAfwAe9mpsnYHkYcNIP+9ctrzwJ8pXrNh9g/ZE1okw5gM4949QuQP+57/gpXNgw/J97jrOH8fDJz/MqAGjePrLp3ls6WMmMkanxUsPZiiwSlVzVbUKeAU4O8q2CiQAQSAeiAM2AjnA96q62a33HnCuWz4beNEtvw6cWuv1GHtHxm234e/WlYIJ96DV1bE2p2Xx+eBn58F1S2DkA7Duc3j2JHhjPOxYt09dB3wBJp8wmQsPvpA/r/gzUz6dQlhbdgaQYbQHvBSY/YGfIs7z3WuNOdcNeb0uIn0BVHUR8CFQ4B7vqOpKYBVwiIhkiUgA+E+gdtuAuuepag2wA2hHS9LbHv70dHpPnEjlypVs+dOfY22ONwTi4fjr4MYvnc+vX4cnj4b37oOKHXvdrU983HXsXVxx2BW89v1rTFg4gZpwTQsabhhtHy8FpinvoXGsYDaQpaqH43gjLwKIyAHAQKAPjnCMEJGTVXUbMB54FVgArAFqf2ujeR4iMk5ElojIks2bNzfRxIgk7bTTSB05ksKnnqIyLy/W5nhHYlcYOcXxaAb+ChY+BtOOggWPwYavnWzOe4iIcNPRN3H9UdczO3c2v5v/O6pDHcwTNIxm8FJg8qn3LsARi/WRFVR1i6pWuqczgMFu+RzgU1UtUdUS4G1gmNtmtqoeq6rHAd8BPzR+nuvdpANbGxulqs+r6hBVHdKzZ88WGGbHp/c9E5CEBAruuptQUVGszfGWrv3h3Jkwbh5kHArv3wfPngCPHgh/uxyWvujkP4sSEWHc4eP43TG/Y+7audzw4Q1U1FR4Zb1htCm8FJjFwIEiki0iQeBCYFZkBRHJjDgdBax0yz/ivNQPiEgcMLz2noj0cj+7AtcAM902s4DaeaHnAR+ovV1tEQI9e9J74kTKly0j96xfUvzee7E2yXv2Owoumw03r4Czn4YDToW1n8DsG+CJI+Dxw2HW9bD8dSjZvSd86aBLufe4e/l43ceMf288pdWlrTAIw4gtnq6DEZEzgccBP/CCqj4gIvcDS1R1log8iCMsNTjexnhV/dadgfY0cDJOmGuOqv7W7fOvwBHuI+5X1Vfc6wnAS8BRbl8Xqmpuc/bZOpg9o/zrFRRMmEDlt9+Sevrp9J5wN4EePWJtVuuhCoXfQ+5HkPcR5C1wZqEBZBwG2cMhZzj0Px7im06z81buW9y98G4O7X4oT//iadLj01txAIbRMthCyygwgdlztLqaLX98gcLp05GkJDLuvIP0s8/u8CuXmyRU42zjnDfPEZ0fP3VS0vgCsP/gesHpc4wzmcDl/R/f57aPbiMnPYfnTnuO7ok2F8VoX5jARIEJzN5TmZtLwd0TKP/iC5JPPJHM+ya17S2XW4Pqcvjp3/UezvovQMMQlwT9jnPEJns49D6cTwo+5cYPb6R3cm9mjJxB7+TesbbeMKLGBCYKTGD2DQ2H2fa/f2WTu39Mr5tvputvLkZ8niaIaD+Ub4e1H9cLzuZvneuJXSHrJJZmHMC1696iS0JXZoycSd/Uvs33ZxhtBBOYKDCBaRmq162jYNJ9lC5YQOJRR5E5ZTLxA1o+NX67p6gA8uY7YpP7ERTl83UwyNWZGcT745lx8BXkDDoXUs2bMdo2JjBRYALTcqgqRbNmsfH3DxIuK6PHtdfQ/corkTjvti5u16jC1lzI/ZDvV81hXPlKwijPb9jEIekD6t/fZJ0ICTYRwGhbmMBEgQlMy1NTWMiGBx6g+O05xB98MJkPPEDiYYfG2qw2z5rtuYx95wpKq4p5RntxxE9fQnUZiM+ZMp09HHJOgb7HQlwHyW5ttFtMYKLABMY7it97jw333U/Nli10v+Jyelx3XcdJ++8R60vWM+bdMRSWF/LU8D8wtEbr39/kLwENQSDBEZmc4ZB9Cux3ZIttN2AY0WICEwUmMN4SKipi0yOPsv1vfyOufz8y759M8rEdYAMzD9lctpmx744lvySfx055jJP7nOzcqCx2FnrmznNEZ5Ob4To+3Qmj5ZziiE6Pg6AzThk3WhUTmCgwgWkdSj/9lIJ7JlL90090ueACet16C/7UphciGrCtYhtXzb2KH7b/wNSTp3Ja/9N2rlSy2V3s6U4Y2L7WuZ7Su346dM5wSO/TusYbnQITmCgwgWk9wuXlbJ72JFtffNFJPXPvvaSO+HmszWqzFFcVc81717CscBmTT5jMqAGjmm+wNa9ebPLmQ1mhc73bgHrvJuskSOrmtelGJ8AEJgpMYFqf8mXLKLh7ApU//EDaWWeRcfddBLrZl15TlFWXccOHN/Dvgn8z4dgJXHDIBdE1DIdh0zdOOC3vIye0VlUCCGQeXj9hoN9xEEzybgBGh8UEJgpMYGKDVlVROHMmhc88iz85mYy77yLtl7/snOlmdkNlqJJb593KvPx5/Hbwb7n8sMv3vJNQNaxbWj9h4KfPIFwN/iD0GVofUtv/aPDbtHJj95jARIEJTGyp/OEHCibcQ/lXX5E8/GQyJ00iLjNz9w07GdXhau5acBdz1szh6iOu5pojrtk3Ma4qhbWL6nOobVgOKARTnUSdWSdCnyGQeaR5OEaTmMBEgQlM7NFQiG0vv8ymPzyO+Hz0uvUWulxwgaWbaUQoHOK+Rffxxqo3uHTQpdw25LaW8/jKtkZkGJjnLAAFED/0GgR9BsP+Q5wEnj0PtmnRhglMNJjAtB2q8vPZMHEipZ8sInHIYDLvn0x8TnaszWpThDXM1MVTeXnly5x30HlMOHYCfi++7Es2OSG1dUud9TfrPq/fliCY6qy96eMKzv5DIM28zs6GCUwUmMC0LVSVHX9/g40PP4xWVNDjuuvofvloSzcTgary5BdPMmP5DM7KOYspJ0wh4At4+9BwGLaudsVmiSM8G5ZD2N2tPG1/5/3N/kPqQ2vxKd7aZMQUE5goMIFpm1Rv2sTGyVMonjuX+EED2W/KFBIGDYq1WW2Kmctn8sTnTzCi7wgeGf4IQX+wdQ2oroANy1zRWeoIT+1W0uKDngMbhtZ6DbTQWgeiTQiMiJwOPIGzo+VMVX2o0f3RwCPAOvfSU6o60703FTgLZ1vnucCNqqoichFwF85Ol+uBS1S1UEQmAWOB2v1r71LVfzVnnwlM26bonXfZMHkyoW3b6H7llfS49hp88fG7b9hJeHnlyzz02UMcv9/xPP7zx0kMJMbWoNJCJ5y2bkm98FRsd+7FJTs51fY/2g2vDYH0Tr5/UDsm5gLjbnv8PXAakA8sBi5S1W8i6owGhqjqdY3aHo8jPG6eDBYCd7qf64FBrqhMBcpUdZIrMCWq+mi0NprAtH1C27ezceoj7Pj73wlmZZH5wBSSBg+OtVlthjd+eINJiyZxZM8jmX7qdFKCbSg0VZsxOjK0VrDMmSINkJrpvscZ7IjOfkftcqtpo20RrcB4GbwdCqxS1VzXoFeAs4Fvmm3loEACEAQEiAM2umUBkkVkC5AGrGp50422gr9LF/b7/QOknXUmGybey9rfXELXiy+i529/iz+lDX2ZxohzDjyHxEAidy64k7HvjuXZ054lPb6NpPcXge4DnOMId5FoTaXz/qZuAsES+PaftQ2g5yFuaM0Nr/UaBH6P3zEZnuGlB3MecLqqjnHPLwWOjfRWXA/mQZyw1vfAzar6k3vvUWAMjqA8pap3R/T7AlAK/AD8XFVDrgczGigClgC3qOq25mw0D6Z9ES4tZfO0aWz9y0sEevcmc9K9pAwfHmuz2gTzfprHLfNuoX96f54/7Xl6JPaItUnRU7Z159Ba+VbnXlySM2mgQWitjyX0jDFtIUR2PvAfjQRmqKpeH1GnO05Yq1JErgZ+raojROQAnHc3tbkx5gK3A4uAOcA4IBd4EtigqlNEJAMoxPF+JgOZqnpFE3aNc9vTr1+/wWvXrvVg9IaXlH3xBQX33EPVqtWkjfoVGXfeSaBr11ibFXM+LfiUGz64gV5JvZg5cia9k9vpzpiqsC0P8pc2DK2FKp37KRk7h9ZsU7ZWpS0IzHHAJFX9D/f8TgBVfXAX9f3AVlVNF5HbgARVnezemwhUAB8CD6nqqe71k4E7VPXMRn1lAf9U1cOas9E8mPZLuKqKLc8+R+Hzz+NPS6P3hLtJPeOMTp9u5otNX3DNe9eQFkxjxsgZ9EvrF2uTWoaaKti43PF0akNrW2qj4+JsU1C3NmcwZBxqaW88pC0ITAAn7HUqziyxxcDFqroiok6mqha45XOA21V1mIhcgDMj7HScENkc4HFgqXscrqqbRWQykKSqtzTq62accNyFzdloAtP+qfjuOwrunkDF11+TMmIEve+dSFxGRqzNiinfbPmGq+ZeRZwvjhkjZzCgy4BYm+QN5dvc0FrE+5yyLc69QIIbWhtcP126Sz8LrbUQMRcY14gzcYTBD7ygqg+IyP3AElWdJSIPAqOAGmArMF5Vv3W9madxZpEpMEdVf+v2eTVwI1ANrAVGq+oWEXkJONKtvwa4qlZwdoUJTMdAa2rY+peX2DxtGhII0Ou22+hy/nmdOt3Mqm2rGDd3HNXhap477TkGde8E64hUnX1xarMPrFsCBV9BTYVzP7ln/eSBPoNhv6MhsUtsbW6ntAmBaeuYwHQsqtaupeCeiZR99hlJQ4eSOfl+gv37x9qsmPFj0Y+MeXcMxVXFPPOLZziy15GxNqn1CVXDxhXuBAL3nU7h9/X3ux/ohNa69IdgsnukuJ9JEWX3elwSxCV2ek/IBCYKTGA6HqrK9r/9jU1TH0Grq+l5w/V0u+wyJNA5p7puKN3AmHfHsKlsE9NGTGNY5rBYmxR7KnbUezi1IbaSjXvQgTQSnl0IUQOxaupw78W57QOtnI1hHzCBiQITmI5L9caNbLjvfko++ICEQw8l84EpJBxySKzNigmF5YWMfXcsPxb9yH+f8t+c0veUWJvU9giHoLrM2cpgp6Mk4l5Jw+tVTVyvLqs/D1VFb4MvLkJ4mhCouCaEbLcCl+xJih4TmCgwgenYqCrFc+awYfIUQkVFdB87hh7jx+MLtp+/FFuK7RXbufq9q/lu63c8eNKDnJ59eqxN6hzUVEF1rSCV7SxQTQpXE/XqhMs913D0NgQSmxaiw86Doy/dq2G1hZX8hhFTRIS0M84gadgwNj30EFueeZbid+eSOXkySUcfFWvzWpUuCV2YOXIm175/LbcvuJ3ymnLOOfCcWJvV8QkEnSOxBddpqToTF5rytOq8qKauNxKu2skPHmIejHkwnYaSBQsouPdeago20PWSS+h10434kpNjbVarUl5Tzk0f3sQn6z/hjqF38JuBv4m1SUY7JFoPpvPO4zQ6HSknnUTOrNl0vfhitr30Erm/GkXJwo9jbVarkhhI5MkRTzKi7wge+uwhZi6fGWuTjA6MCYzRqfCnJNP7ngn0f/l/kPh4fhozhvV33Elo+/ZYm9ZqBP1BHj3lUc7MPpMnPn+CaZ9PozNHMgzvMIExOiVJgweT/Y836H7VVeyYPZvVZ/2SojnvdJov2jhfHL8/8fece+C5zFg+g4cXP0x4T14cG0YUmMAYnRZffDy9br6J7Nf/RlxGButuuol1N9xA9aZNsTatVfD7/Nx73L1cOuhSXl75MpM+mUQoHIq1WUYHwgTG6PQkDBxI1muv0uvWWyiZv4Dcs37J9v/7v07hzYgItw25jauPuJo3Vr3BHQvuoLp2QzDD2EdsFpnNIjMiqMzLY8M9EylbsoSk44bR+667CA4Y0Cnymv3p6z/x2NLH6JnYk6S4JHziwy/+Jj994sPv2/meX/x115tr32TbJq7t9Onb+XmNr0X9vIi2SYEk0uLTiPfbltzRYAsto8AExmgKDYfZ/tprbHrkUcKlpUhiIsHsLOKzcwgOyCE+J4dgTg7BrKwOt2jzX7n/YsG6BYQ0RCgcIqxhQrqLT/d+5LUarSEc3nWbunKjOrXlWJMYSCQtmEZ6fLpzBJ3PtPi0unLk9fT4dNKCaSQGEjvVVhEmMFFgAmM0R/XGjZTM+4iq3NVU5uZRlZtL9bp19RV8PuL69nGEJyeH+AE5BLOdT3+6bYC1p6jqToK1K1HblaDtsm24UZtG18trytlRucM5qnbUlYuqithRuYPtldubDR3G+eKiEqS0+LQG5ylxKe1SmGwlv2HsI3EZGXS94NcNroXLy6las4bK1blU5eZSmet8ln7yCVpVn3fK3717nacTKTyB3r07RbhtbxARJ3SFnzja1mZhqkpFqGIn4dmVIBWUFPBt1bfsqNxBeU35Lvv1i7/OY2pKlBoLUm05NZiK34McYy2NeTDmwRgtgIZCVK9bR+Xq1VTl5lGZW/uZS3jHjrp6deG2nAEEc7LrPjtiuM1wqApVNRSkXYhS5PWiyiKKq4ub7Tc1mBqdIEXeC6YT1wI7fbaJEJmInA48gbPh2ExVfajR/dHAIzg7XgI8paoz3XtTgbNwZrrNBW5UVRWRi4C7cDYWWw9coqqFItINeBXIwtlw7Nequq05+0xgDK9RVUJbtzrezupcqvJy67yf6vXr6yvWhtsaCU98joXbOis14RqKq4qjFqS6clVRs++zkgJJpMenc9EhF3H5YZfvlW0xD5G5u1JOB04D8oHFIjJLVb9pVPVVVb2uUdvjgROAw91LC4HhIrIQR7AGuaIyFbgOmATcAbyvqg+JyB3u+e3ejM4wokNECHTvTqB7d5KOOabBvXBZWX24LUJ4ShcuRKvr4/3+Hj2Iz86OmGAwgPicbAKZme0yfm9ER8AXoGtCV7om7FmizLCGKakuqReeRgJVW85MzvTI8nq8fAczFFilqrkAIvIKcDbQWGCaQoEEIAgIEAdsdMsCJIvIFiANWOW2ORs4xS2/CMzDBMZow/iSkkgYNIiEQQ23M9aaGjfc5gpPbi5Vq3Mp+tfbhIuK6upJUhLxWVkEBziCUys8wf79EQu3dVp84iMtmEZaMA1SY2uLlwKzP/BTxHk+cGwT9c4VkZOB74GbVfUnVV0kIh8CBTiC8pSqrgQQkfHAcqAU+AG41u0nQ1ULAFS1QER6eTEow/AaCQQI9u/vbvf887rrqkpoy5a6iQW1wlO2dAlFs2fXd+D3E+zTZ6cJBsGcHPxpaa0/IKPT4qXANOW7N37hMxv4q6pWisjVOJ7HCBE5ABgI9HHrzXVFaBEwHjgKyAWeBO4EpkRtlMg4YBxAv379oh+NYcQYESHQoweBHj1IHjq0wb1waSmVa9ZEzGzLoyp3NSULF0JkuK1nj/r1PBHregK9e1u4zWhxvBSYfKBvxHkfnJfydajqlojTGcDDbvkc4FNVLQEQkbeBYUC52261e/01nHctABtFJNP1XjKBJhNKqerzwPPgvOTf69EZRhvCl5xM4qGHknjooQ2ua00N1fn57joedz3P6tUU/fMtwsX1s5QkKcl5z5OTQ9x+++FLScafkoIvJQVfckrD89RU/MnJFoYzdouXArMYOFBEsnFmiV0IXBxZoVYQ3NNRwEq3/CMwVkQexPGEhgOPu/0MEpGeqroZZwJBbZtZwGXAQ+7nm14NzDDaCxIIEMzKIpiVBSMahdsKCx3BiZhgULZkCTWbNkFo90kvJRh0BCfFFaDklIbnKSn4UlKbEKsU/CnJdXUlPt68pw6KZwKjqjUich3wDs405RdUdYWI3A8sUdVZwA0iMgqoAbYCo93mrwMjcN61KDBHVWcDiMh9wHwRqQbWRrR5CHhNRK7EEajzvRqbYbR3RIRAz54EevYk+diG4TZVRSsqCJeUECopIVxSSri0pOF5SQnhkuJG5yVUb9hQVw6VlDQIz+2SQAB/cjK+1NTmxSq50XlqagPvShI7V7qW9oAttLR1MIbhGeGqqnrBKS5uRqxKCJeWEKo9Ly4mVFp/Xysrd/8wn68ZgUpt5Fk19KYCGRn2HmoPiPk6GMMwDF8wiK9bN+jWbZ/60aoqQqWlhEtLG3hIu/SmSp37oe3bqc7Pd+qXlqJlZbt8Rl2WhaxsgtnZzkLXbDfLQlLSPtnfWTGBMQyjzSPBIIFgELru2aLDxmhNTZ1IhWq9qeJiqgsKnNl3eWsoX7aMorffhojoTiAzk/jsLILZOY74ZGc5s+8yMiy3XDOYwBiG0WmQQAB/ejr+9PRm02mGKyqoWvsjVXnuJIi8PKry1rDjH/8gXFpa319iIsGsLMfTcY/4HPN6ajGBMQzDaIQvIYGEgw8i4eCDGlxXVWo2b3bWGa3Joyovj8rcvKa9nt69XbFxpn87ewpld6qM2iYwhmEYUSIixPXqRVyvXiQPa5iYpIHXsyavLuS24803d+H1NAq5ZWXhS05u3QF5jAmMYRhGC7BbrydvDVV5uY7Xk5dH+bLlFL09Z9deT8REg/bq9ZjAGIZheEgDr6fRmqNwZSVVa9bu7PXMmkW4pKS+j4QE5/1OdlbDkFsb93pMYAzDMGKELz4+Cq+nfqLBrrwe5/1OToOJBm3B6zGBMQzDaGPs1utZu3aniQZNej1ZWTuH3FrR6zGBMQzDaEf44uNJOOggEg7a2eupzy8X4fUs/3pnrycjg26XXUb3K/ZuR8toMYExDMPoADSXX67O64mYaBDo2dNzm0xgDMMwOji78no8f26rPs0wDMPoNJjAGIZhGJ5gAmMYhmF4ggmMYRiG4QmeCoyInC4i34nIKhG5o4n7o0Vks4h86R5jIu5NFZEVIrJSRKaJQ2pE3S9FpFBEHt9dX4ZhGEbr49ksMhHxA9OB04B8YLGIzFLVbxpVfVVVr2vU9njgBOBw99JCYLiqzgOOjKi3FPh7c30ZhmEYscFLD2YosEpVc1W1CngFODvKtgokAEEgHogDNkZWEJEDgV7Aghaz2DAMw2gxvBSY/YGfIs7z3WuNOVdElonI6yLSF0BVFwEfAgXu8Y6qrmzU7iIcj0Wb68swDMOIDV4utJQmrmmj89nAX1W1UkSuBl4ERojIAcBAoI9bb66InKyq8yPaXghcuru+djJKZBwwzj0tEZHv9nRgLj2Awr1s216xMXcObMydg30Zc/9oKnkpMPlApBfRB1gfWUFVt0SczgAedsvnAJ+qagmAiLwNDAPmu+dHAAFVXRpFXw1Q1eeB5/diPA0QkSWqOmRf+2lP2Jg7BzbmzkFrjNnLENli4EARyRaRII7HMSuygohkRpyOAmrDYD8Cw0UkICJxwPCIe+CEx/4aZV+GYRhGDPDMg1HVGhG5DngH8AMvqOoKEbkfWKKqs4AbRGQUUANsBUa7zV/HCW8txwmrzVHV2RHd/xo4s9Ejd9WXYRiGEQOk4TtyNBfqCwAABcFJREFUI1pEZJwbbus02Jg7BzbmzkFrjNkExjAMw/AESxVjGIZheIIJzG6IIt3N1SKy3E1Ps1BEBsXCzpZkd2OOqHeeiKiItPvZN/uS1qi9Es3PWUR+LSLfuGmb/re1bWxpovg5/yHiZ/y9iGyPhZ0tSRRj7iciH4rIF+46wsbvt/ceVbVjFwfO5ITVQA5OVoGvgEGN6qRFlEfhTEiIue1ejtmtl4ozbfxTYEis7W6Fn/No4KlY29rKYz4Q+ALo6p73irXdXo+5Uf3rcSYnxdx2j3/OzwPj3fIgYE1LPd88mObZbbobVS2KOE1m58Wk7Y1oU/xMBqYCFa1pnEfsS1qj9ko0Yx4LTFfVbQCquqmVbWxp9vTnvNNyiHZINGNWIM0tp9NoveK+YALTPFGluxGRa0VkNc4X7g2tZJtX7HbMInIU0FdV/9mahnnIXqc1asdEM+aDgINE5GMR+VRETm8167wh2p8zItIfyAY+aAW7vCSaMU8CLhGRfOBfOJ5bi2AC0zzRpLtBVaer6gDgdmCC51Z5S7NjFhEf8AfgllazyHuiTWuUpaqHA+/hpCJqz0Qz5gBOmOwUnL/mZ4pIF4/t8pKofp9dLgReV9WQh/a0BtGM+SLgz6raB2d94Uvu7/k+YwLTPLtNd9OIV4D/9NQi79ndmFOBw4B5IrIGJ4XPrHb+oj+qtEaqWumezgAGt5JtXhHN/+184E1VrVbVPOA7HMFpr+zJ7/OFtP/wGEQ35iuB16Au0XACTp6yfcYEpnmiSXcT+Qt3FvBDK9rnBc2OWVV3qGoPVc1S1Sycl/yjVHVJbMxtEfYlrVF7ZbdjBv4B/BxARHrghMxyW9XKliWaMSMiBwNdgUWtbJ8XRDPmH4FTAURkII7AbG6Jh3uZ7LLdo9Glu7lORH4BVAPbgMtiZ/G+E+WYOxRRjrlDpSKKcszvACNF5BsgBNymDZPKtiv24P/2RcAr6k6ras9EOeZbgBkicjNO+Gx0S43dVvIbhmEYnmAhMsMwDMMTTGAMwzAMTzCBMQzDMDzBBMYwDMPwBBMYwzAMwxNMYAzDI0Rkkojc2gbsWOOuYzGMVsUExjAMw/AEExjD2ANEJFlE3hKRr0TkaxG5INJDEJEhIjIvoskRIvKBiPwgImPdOpkiMt/dc+RrETnJvf6MiCxx9165L+KZa0Tk9yKyyL1/tIi8IyKrReRqt84pbp9vuPu3PNtUPikRuUREPnOf/ZyI+L389zI6NyYwhrFnnA6sV9UjVPUwYM5u6h+Ok0LoOGCiiOwHXAy8o6pHAkcAX7p171bVIW6b4SJyeEQ/P6nqccAC4M/AeTh54O6PqDMUZ1X2z4ABwH9FGuKmAbkAOMF9dgj4zR6M3TD2CEsVYxh7xnLgURF5GPinqi4QaSphbR1vqmo5UC4iH+KIwGLgBRGJA/6hqrUC82sRGYfze5mJs/nTMvdebRqT5UCKqhYDxSJSEZHh+DNVzQUQkb8CJwKvR9hyKk6SzsWuzYlAe9/jxWjDmMAYxh6gqt+LyGCctOYPisi7OPnJaqMBCY2b7NyFzheRk3E8m5dE5BEcz+RW4BhV3SYif27UV20m53BEufa89vd4p2c1OhfgRVW9czfDNIwWwUJkhrEHuCGuMlX9H+BR4GhgDfXp+89t1ORsEUkQke44+6osdjez2qSqM4A/un2kAaXADhHJAM7YC/OGullzfTihsIWN7r8PnCcivdyxdHNtMQxPMA/GMPaMnwGPiEgYJ4P2eJxQ0x9F5C7g343qfwa8BfQDJqvqehG5DLhNRKqBEuD/qWqeiHwBrMBJif/xXti2CHjItXE+8EbkTVX9RkQmAO+6IlQNXAus3YtnGcZusWzKhtEBEJFTgFtV9ZextsUwarEQmWEYhuEJ5sEYhmEYnmAejGEYhuEJJjCGYRiGJ5jAGIZhGJ5gAmMYhmF4ggmMYRiG4QkmMIZhGIYn/H9TzqLaUCZWWQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f761a168da0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch3_1.best_score_, gsearch3_1.best_params_))\n",
    "test_means = gsearch3_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch3_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch3_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch3_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch3_1.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第五步：调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58688, std: 0.00355, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58742, std: 0.00345, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58735, std: 0.00318, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58711, std: 0.00316, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58739, std: 0.00314, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58748, std: 0.00333, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       " -0.5868815472598764)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=270,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=4,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([133.48940339, 127.07015996, 133.1488945 , 132.2728065 ,\n",
       "        131.23091841, 134.4953753 ]),\n",
       " 'mean_score_time': array([1.97586079, 0.36381021, 0.30010767, 0.30629625, 0.41504006,\n",
       "        0.63366952]),\n",
       " 'mean_test_score': array([-0.58688155, -0.58741703, -0.58734787, -0.58710569, -0.58739268,\n",
       "        -0.58747614]),\n",
       " 'mean_train_score': array([-0.53670592, -0.53742228, -0.53866492, -0.53796216, -0.53854411,\n",
       "        -0.53960012]),\n",
       " 'param_reg_alpha': masked_array(data=[1.5, 1.5, 1.5, 2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.5, 1, 2, 0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'rank_test_score': array([1, 5, 3, 2, 4, 6], dtype=int32),\n",
       " 'split0_test_score': array([-0.58102122, -0.5821575 , -0.58203751, -0.58222059, -0.5823366 ,\n",
       "        -0.58240392]),\n",
       " 'split0_train_score': array([-0.53800845, -0.53884755, -0.53983993, -0.5390383 , -0.54013746,\n",
       "        -0.54132242]),\n",
       " 'split1_test_score': array([-0.58547367, -0.58664922, -0.58628953, -0.58564837, -0.58622943,\n",
       "        -0.58562077]),\n",
       " 'split1_train_score': array([-0.53664201, -0.53749358, -0.53887646, -0.53852752, -0.53902876,\n",
       "        -0.53940378]),\n",
       " 'split2_test_score': array([-0.58689691, -0.5859189 , -0.58724273, -0.58656218, -0.58698436,\n",
       "        -0.58723276]),\n",
       " 'split2_train_score': array([-0.53671167, -0.53688967, -0.53831259, -0.53730132, -0.53822747,\n",
       "        -0.53905666]),\n",
       " 'split3_test_score': array([-0.59007352, -0.59066436, -0.59026139, -0.59074463, -0.59033899,\n",
       "        -0.59051387]),\n",
       " 'split3_train_score': array([-0.53603666, -0.53694988, -0.53826825, -0.53760422, -0.537632  ,\n",
       "        -0.53884055]),\n",
       " 'split4_test_score': array([-0.59094364, -0.59169646, -0.59090929, -0.59035365, -0.59107514,\n",
       "        -0.59161061]),\n",
       " 'split4_train_score': array([-0.53613083, -0.53693074, -0.53802738, -0.53733946, -0.53769487,\n",
       "        -0.53937721]),\n",
       " 'std_fit_time': array([ 3.11811775,  5.2005563 , 10.46550856,  4.22538939,  0.97133617,\n",
       "         5.39643181]),\n",
       " 'std_score_time': array([0.90968147, 0.05121015, 0.00763804, 0.02132875, 0.13017851,\n",
       "        0.32752924]),\n",
       " 'std_test_score': array([0.00355004, 0.00344538, 0.00317813, 0.00316463, 0.00314083,\n",
       "        0.00333313]),\n",
       " 'std_train_score': array([0.00070418, 0.00074632, 0.00065013, 0.00069718, 0.00094096,\n",
       "        0.00088621])}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.586882 using {'reg_alpha': 1.5, 'reg_lambda': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl4VdXV+PHvyhySkAAJQwgQkFkFlEgRVAQnQMSxVhzBKvoqr9rWWq3D6w+1KrVWbbUOdcDZihOgiGJBREUJgiiEIcxhDAQCATKv3x/nRG5uphvIHZKsz/PkyT337nPuOoGblb3POnuLqmKMMcY0tLBgB2CMMaZpsgRjjDHGLyzBGGOM8QtLMMYYY/zCEowxxhi/sARjjDHGLyzBGGOM8QtLMMYYY/zCrwlGREaKyCoRyRaRO6t5fbyI5IrIUvfrOo/XpojIchHJEpGnxJHg0XapiOwSkSfqOpYxxpjAi/DXgUUkHHgaOAvIARaJyHRVXeHV9B1VneS17xBgKNDPfWoBMExV5wEDPNotBt6v7Vi1SU5O1vT0dF+bG2OMARYvXrxLVVPqaue3BAMMArJVdR2AiLwNnA94J5jqKBADRAECRAI7PBuISA+gLfDVkQaYnp5OZmbmke5ujDHNkohs9KWdP4fIOgKbPbZz3Oe8XSwiy0Rkmoh0AlDVb4G5wDb3a7aqZnntNw6nx6K1HcubiEwUkUwRyczNzT3CUzPGGFMXfyYYqeY575k1ZwDpqtoPmANMBRCR7kAfIA0nKY0QkdO89r0MeKuuY1UJQPV5Vc1Q1YyUlDp7eMYYY46QPxNMDuDZi0gDtno2UNXdqlrkbr4ADHQfXwgsVNUCVS0AZgGDK/YTkf5AhKou9uFYxhhjgsCf12AWAT1EpCuwBafHcblnAxHpoKrb3M2xQMUw2CbgehF5GKcnNAx4wmPXcVTuvdR2rHopKSkhJyeHwsLCI9nd+FlMTAxpaWlERkYGOxRjTB38lmBUtVREJgGzgXDgJVVdLiKTgUxVnQ7cIiJjgVIgDxjv7j4NGAH8hDOs9qmqzvA4/KXAaK+3rOlY9ZKTk0NCQgLp6emIVDfKZ4JFVdm9ezc5OTl07do12OEYY+ogzXnBsYyMDPWuIsvKyqJ3796WXEKUqrJy5Ur69OkT7FCMabZEZLGqZtTVzu7kr4Yll9Bl/zbGNB7+vAZjjDEmVJSVwr4cyFsPe9ZD5yHQtrdf39ISjDHGNBXFB2HPBieBVCSSiu97N0F56eG2Ix+xBGP8a968eTz22GPMnDnzqNrUZuXKlUyYMIEffviBhx56iNtvv73aduPHj+fLL78kMTERgFdeeYUBAwZU29aYZkkVDuZVn0Dy1kPB9srtoxOhdTq07wd9z4dWXaF1V+d7y1S/h2sJJsSpKqpKWFjjvVzWunVrnnrqKT788MM62/71r3/lkksuCUBUxoSo8jLYt6X6BLJnAxTtq9w+oYOTMLqfUTmBtO4Ksa0giNctLcHU4v/NWM6KrfvqblgPfVNb8n/nHVtrmw0bNjBq1CiGDx/Ot99+y2233cazzz5LUVERxxxzDC+//DLx8fF88skn/P73vyc5OZkTTzyRdevW1djL+P7777nttts4dOgQsbGxvPzyy/Tq1atSm/vvv5+1a9eyZcsWNm/ezB133MH1118PQEFBAZdccgk///wzAwcO5PXXX0dEmDx5MjNmzODQoUMMGTKE5557rsqF+LZt29K2bVs+/vjjo/jJGdOElByCPRtrHsoqKz7cNiwSkjo7CaPTryonkKQuENUieOdRB0swIWrVqlW8/PLLTJ48mYsuuog5c+YQFxfHo48+yuOPP84dd9zBDTfcwPz58+natSvjxo2r9Xi9e/dm/vz5REREMGfOHP785z/z3nvvVWm3bNkyFi5cyIEDBzjhhBM499xzAViyZAnLly8nNTWVoUOH8vXXX3PKKacwadIk7rvvPgCuuuoqZs6cyXnnncezzz4LwI033liv87777ruZPHkyZ5xxBo888gjR0dH12t+YkFFlKGvD4e39Wyu3jUpwhrLa9oXe51buiSSmQVh4MM7gqFmCqUVdPQ1/6tKlC4MHD2bmzJmsWLGCoUOHAlBcXMzJJ5/MypUr6dat2y83HI4bN47nn3++xuPl5+dzzTXXsGbNGkSEkpKSatudf/75xMbGEhsby/Dhw/n+++9JSkpi0KBBpKWlATBgwAA2bNjAKaecwty5c5kyZQoHDx4kLy+PY489lvPOO6/eiQXg4Ycfpn379hQXFzNx4kQeffTRX5KXMSGnvNxJFNUOZa2HwvzK7ePbOQmj27CqQ1kt2gR1KMtfLMGEqLi4OMC5BnPWWWfx1luVZsZhyZIl9Trevffey/Dhw/nggw/YsGEDp59+erXtvIe3KrY9exLh4eGUlpZSWFjITTfdRGZmJp06deL+++8/qil2OnTo8Mt7TZgwgccee+yIj2VMgygtqnkoa89GKCs63DYsAhI7OQmj48DKCaRVOkTFBe00gsUSTIgbPHgwN998M9nZ2XTv3p2DBw+Sk5ND7969WbduHRs2bCA9PZ133nmn1uPk5+fTsaOzWsIrr7xSY7uPPvqIu+66iwMHDjBv3jweeeQRVq9eXW3bimSSnJxMQUEB06ZNO6oL9Nu2baNDhw6oKh9++CHHHXfcER/LGJ8d2lv9xfS89c7Fds9J4CPjnISR3BN6nuM1lNUJwu1Xqif7aYS4lJQUXnnlFcaNG0dRkfPX0oMPPkjPnj155plnGDlyJMnJyQwaNKjW49xxxx1cc801PP7444wYMaLGdoMGDeLcc89l06ZN3HvvvaSmptaYYJKSkrj++us5/vjjSU9P56STTvrlNc9rMNu3bycjI4N9+/YRFhbGE088wYoVK2jZsiWjR4/m3//+N6mpqVxxxRXk5uaiqgwYMOCXYxhzVMrLnfLdmoayDu2p3D4uxUkY6UOrDmXFpTTJoSx/sbnIqpmLrLHMc1VQUEB8fDyqys0330yPHj343e9+d8THu//++4mPj6/xPpVQ0Zj+jUyAlBY71VfVDmVtgFKPoVsJOzyU5Z1AWqVDdEKwzqLR8HUuMuvBNGIvvPACU6dOpbi4mBNOOIEbbrgh2CEZ4z+F+2q4wXCDMwWKlh9uGxHrJIzWx0D3M53EUZFIkjpDuC33EAjWg2nEPZjqvPzyyzz55JOVnhs6dChPP/10kCJqeI3938jUQBUKdtQ8lHVwd+X2LdpU0wNxv8e3s6EsP7IeTDM1YcIEJkyYEOwwjKleWYnXUNaGykNZJQcPt5UwaJnm3B/Se0zVRBLTMkgnYXxlCcYY07CKCmqeKys/B7TscNuIGGf4qlVX6HZ65QSS1BkiooJ0EqYhWIIxxtSPKhzYVTV55K1zHh/Irdw+tpWTMNIy4Phfew1ltYdGPM+eqZ0lGGNMVWWlkL+58lCW53QnxQUejQVadnQSRs+RVYeyYpOCdBIm2CzBGNNcea8dUtEDyVvvJBfPtUPCow4PZXnfH5LUGSJjgnUWJoRZgmnmArEezBtvvMGjjz4KQHx8PP/617/o37//ER3L1EN91w6JSXQSRuoAOPbCyj2RhFQbyjL1ZgkmxDWF9WC6du3Kl19+SatWrZg1axYTJ07ku+++C3ZYTUO91w5JdRJG9zOd6izPnkiL1sE4A9OEWYKpzaw7YftPDXvM9sfDqEdqbdLU1oMZMmTIL48HDx5MTk7OkfzkalVaVs4d7y1jf2EpqYkxdEiKpUNiDKnu93YtY4gMb6RJura1Q/ZshHKPmbHDIqFVFydhdB5cOYG06gKRscE7D9PsWIIJUU11PZgXX3yRUaNGNcSPqJK3F23m/R+20DU5joVrd7O/qLTS62ECKQnRdEiMJTUphg6JTuLpmBRLh6RYUhNjSI6PJiwsSDfn1WftkOiWzvWQdsdWvT+kZcdGu3aIaXoswdSmjp6GPzXF9WDmzp3Liy++yIIFC472x1P53A6V8Pjnq/lV19a8PXEwIsL+whK25Reyde8htuUXsm3vIbbmF7It/xArt+3nvyt3UlhSXuk4keFCu5YxpCbG0sFNQp7JKDUpllYtIqv00HxS77VD2jsJo9vpVauyWrS2u9RNo+DXBCMiI4EngXDg36r6iNfr44G/Alvcp/6pqv92X5sCnAuEAZ8DtwLxwFceh0gDXlfV2zyOeQnwLnCSqlaeB6YRaWrrwSxbtozrrruOWbNm0aZNm3rFXpd//ncNew4Wc++Yvr/EmxATSUJMJD3bVT9xoaqy92AJW/MPsW2vk3i2eiSiHzbtYXv+NkrKKk+lFB0R9suwW6UElBRDx/gwUtlJ/IHNvq0dktT58P0hlYay0kN6GVxjfOW3BCMi4cDTwFlADrBIRKar6gqvpu+o6iSvfYcAQ4F+7lMLgGGqOg8Y4NFuMfC+x3YCcAvQZK4gN4X1YDZt2sRFF13Ea6+9Rs+ePX08c9+s33WAV77ZwKUDO3Fcx0Sf9xMRWsVF0SouimNTq9+vvFzZdaDocALaW0je7p2U7l5HxN4NxG3fTOviLXRmJ53DdtCBPMLkcEI6JLHsie7IwbhOlHYZSkRyN1q0707rtN7EtOlsa4eYJs+f/8MHAdmqug5ARN4Gzge8E0x1FIgBogABIoEdng1EpAfQlso9mgeAKUBozzdfD01hPZjJkyeze/dubrrpJgAiIiLwnmT0SP3lkyyiwsP4wzkNlLg81g4J27OetnnrabtnPf1rWDtEW7alpGUX9rc4jTWRHdkS1o51pW1ZWdSG1QUxbM0vYteWosN9dBTIIqlFttMDSoypMhyXmhhLu8RooiPsWopp3Pw2m7I7VDVSVa9zt68CfuXZW3GHyB4GcoHVwO9UdbP72mPAdTgJ5p+qerfX8e8DWqrq7e72CcA9qnqxiMwDbq9riKyxz6bc3NeD+Tp7F1f8+zvuGNmLm07v7vsB6rV2SDgkdap+1t5W6RAdX+fbFZWWsSO/yBmOc3tC29yhuYrrQnsPVr0mlhwf7SYdrwTkfm+bEE1EY62MM41aKMymXN1VSO9sNgN4S1WLRORGYCowQkS6A31wrrEAfC4ip6nqfI99LwOuAhCRMODvwPg6gxKZCEwE6Ny5s+9nE4Ka83owZeXKAzNXkNYqlmuHdq3aoD5rh0S2cBJGm+7u/SEeiSSx01GvHRIdEU7nNi3o3Kbm6yoHi0vdYoTCKteF1uUe4Ovs3RRUUxnXrqWbgNxKOO/rQslxQayMM82eP3swJwP3q+o57vZdAKr6cA3tw4E8VU0UkT8CMar6gPvafUChqk5xt/sD76pqT3c7EVgLVEyQ1B7IA8bW1otp7D2Y6jSX9WCW7Ivjzx/8xDNXnMjo4zs494p8MRk2f1/D2iHJNaxg2BXi2zaKqqx9hSVVE1BFb8itmCsqrVoZ174i8Xglog5JTsVc0pFWxplmKxR6MIuAHiLSFWcE+jLgcs8GItJBVbe5m2OBLPfxJuB6EXkYpyc0DHjCY9dxwC9lVaqaDyR7HHcePgyR1URVG+0HrqmvB6OqlKvyt89WMSi9NaOOaw9F++GtcbBhAXQ9FfqcV3UoqwmsHdIyJpKW7SPp1b7myri8A8VOTyi/sMpwXObGPWxfto3S8sp/VMZEhlUuza7mRtWEGFsB0tSf3xKMqpaKyCRgNk6Z8kuqulxEJgOZqjoduEVExgKlOD2O8e7u04ARwE84w2qfquoMj8NfCoz2R9wxMTHs3r2bNm3aNNok01SpKrt372br/jLyDhbzypi+yKE98PrFsO1HuOgF6PfrYIcZNCJCm/ho2sRH11hRV16u7CooYsveQ5XvE3KT0Vdrctm5vwjvgY2E6Iga7w2q+B4TaUUJpjJbMtlriKykpIScnJwa7+cwwaVhkVz9Tjan9+nAYyPbwWsXwu61cOlU6NXwMwQ0RyVl5ezYV1jjjarb9hay+0Bxlf1atYiscg0o1SMRtWsZQ1SEFSU0BaEwRNYoRUZG/nJ3vAk9N762mIOl8KfBMfDSOVCQC1e8C92GBTu0JiMyPIy0Vi1Ia1VzUUJhSRnb86sWJGzbe4icPYf4fn0e+worFyWIuJVxXteAPHtGbRNiCLeihCbDEoxpNL5du5tPl2/noVMiSfnPBc767ddMd+6ENwEVExlOenIc6clxNbY5UFRa6TqQZ0HCmp37mb8ml4PFZZX2CQ8T2iVEV7kG5NkzahMXZZVxjYQlGNMoVJQlD2+5lctXPOzcnzLhE2fCRxOS4qIj6N42ge5tay5K2HeotIb7gw7x05Z8Pluxg2Kvyrio8DC3Ms4jAXmVaSfGWmVcKLAEYxqFaYs3E7f9e56PexyJbA1Xfwhtjgl2WOYoiAiJLSJJbBFJnw7VV/mpKrsPFHuUZ7vFCe5w3Pfr89i+r5Ayr8q42MjwSteAfklAHt/jo+3Xn7/ZT9iEvP2FJSyY9Q6vR/+ViMTOTnJJTKt7R9PoiQjJ8dEkx0dzfFr1lXFl5Uru/qJq7w/aml/I6tW55BZUUxkXE1FtebZnmbZVxh0dSzAm5H3x3vP8rexhytr0QiZMh/iUYIdkQkh4mHMzafvEGKhhco7i0sOVcdVdF1qWk09eNZVxreOiapyqp4P7no12IbsAsARjQlregpc4b/XdbGxxHN2unwmxScEOyTRCURFhdGrdgk6ta6+Mq1SS7VGavTnvIN+t383+airjUuKja5yqJzUxlpSE6GZbGWcJxoSuhf+i9Zw7WUA/evz2Q0suxq9iIsPpmhxH11oq4wqKSqsmIPe60Kod+5m3KpdDJZUr4yLCpJo54yoeO4moTVxUkyxKsARjQo8qfDkF5v2FWWUnsfbUJzkluWEXKTPmSMRHR9CjXQI9alnILv9QSaVrQNs8Zk34cfNeZv9cSHGZV2VcRJg7FFf9iqqpibG0jI1odEnIEowJLaow+25Y+DRzos/gIb2Rz0/vHeyojPGJiJDUIoqkFlH0Ta2+Mq683K2Mq2ay0m35hSxct5sd+4uqVMa1iAqv8d6giu9xIVYZF1rRmOatvAxm3ApLXmNN+uVcv3I0T1x2LLFRVsljmo6wMCElIZqUhGj61VAMWVpWTm5BUZV7gyqq5FZu30/u/qIq+7WMiaj23qCK60HtA1wZZwnGhIbSYvhgIiz/gOKht3P5d4MZ0LkFY/unBjsyYwIuIjzMrVSLBVpV26aiMm7rL/cGVS7TXrp5L3uqWciuTVwUHZJimDS8OyOP6+Df8/Dr0Y3xRfFB+M/VkP05nP0gT+0/m9yCbJ67OqPRjTkbEyi+VMYdKi6rMgRXkYACsSS3JRgTXIX74K3LYOM3cN6T5HS7lOf/9iUXDEjlxM7V/+VmjPFNbFQ43VLi6ZZS99Le/mAJxgTPgd3w+kWw42e4+N9w/CU88uYPhAncMdIu7BvT2FmCMcGxb6uzlsueDXDZm9DzHDI35DFz2TZuPaMHqUmxwY7QGHOULMGYwMtbD6+eDwd3w5XvQfoplJcrk2euoF3LaG4Y1i3YERpjGoAlGBNYO7Pg1QugrMhZy6XjQAA+WLKFZTn5PH5pf1pE2X9LY5oCm6XNBM6WH+Dl0c7j8Z/8klwOFpcyZfZK+qclcsGAjkEM0BjTkCzBmMDYsACmjoXoeLh2FrTr+8tLz365jh37irjvvL62UqExTYglGON/qz+D1y+Glqlw7Wxoffgay5a9h3juy7Wc1z+VgV1aBzFIY0xDswRj/Ovn9+DtcZDSGybMcpKMhymfrgTgTyN7BSM6Y4wfWYIx/rN4Kkz7LaQNci7ox1WeEXnxxj18tHQrE0/rRlqrmu9GNsY0TpZgjH988w+YcQt0P8MpRY6pvNxtebnywMwVtE2I5sZhxwQpSGOMP/k1wYjISBFZJSLZInJnNa+PF5FcEVnqfl3n8doUEVkuIlki8pQ4EjzaLhWRXSLyhNv+RhH5yX1+gYj09X4/EwCq8N+H4LN7oO8FcNlbEFW1dzL9x60s3byXO0b2Drkpxo0xDcNvn2wRCQeeBs4CcoBFIjJdVVd4NX1HVSd57TsEGAr0c59aAAxT1XnAAI92i4H33c03VfVZ9/mxwOPAyAY9KVO78nKYfRd89yyccCWc9xSEVZ1Q72BxKY/MWsnxHRO56AQrSzamqfJnD2YQkK2q61S1GHgbON/HfRWIAaKAaCAS2OHZQER6AG2BrwBUdZ/Hy3HuMUyglJXC9ElOchl8M4z9Z7XJBeD5+evYvq+Qe8dYWbIxTZk/E0xHYLPHdo77nLeLRWSZiEwTkU4AqvotMBfY5n7NVtUsr/3G4fR+fkkkInKziKwFpgC3NNypmFqVFsG0CbD0DTj9LjjnIahhmv1t+Yd49su1nHt8BwZ1tbJkY5oyfyaY6n7DePcqZgDpqtoPmANMBRCR7kAfIA0nKY0QkdO89r0MeKvSwVWfVtVjgD8B91QblMhEEckUkczc3Nx6npKpoviAM91+1nQ452E4/c4akwvAXz9dRbnCnaNstmRjmjp/JpgcoJPHdhqw1bOBqu5W1Yp1P18ABrqPLwQWqmqBqhYAs4DBFfuJSH8gQlUX1/DebwMXVPeCqj6vqhmqmpGSklLfczKeCvPhtYtg3TxnSOzkm2ptvnTzXt5fsoXrTula6yJJxpimwZ8JZhHQQ0S6ikgUTo9jumcDEfFcr3MsUDEMtgkYJiIRIhIJDPN4DZzhsUq9F/eaTIVzgTUNchamegd2wStjYMtiuOQlOPGqWpurKpNnLCc5PpqbhncPUJDGmGDyWxWZqpaKyCRgNhAOvKSqy0VkMpCpqtOBW9yKr1IgDxjv7j4NGAH8hDOs9qmqzvA4/KXAaK+3nCQiZwIlwB7gGv+cmSF/C7x2AezdDOPehh5n1rnLjGXb+GHTXqZc3I94K0s2plkQj2vkzU5GRoZmZmYGO4zGZfdaZ7r9Q3vgiv9AlyF17lJYUsaIx+bRKi6K6ZNOIdwqx4xp1ERksapm1NXO/pQ0vtux3FmFsqwExs+A1BN82u2F+evYml/I478ZYMnFmGbEpooxvslZ7KzlImHOpJU+Jpcd+wp5Zt5aRh3XnsHd2tS9gzGmybAEY+q2fj68OhZik+DaT6Gt7yXGUz5dRVm5cteoPn4M0BgTiizBmNqtmgWvXwKJnWDCp9Aq3eddl+Xs5b0fcphwSjqd21hZsjHNjSUYU7OfpsE7VzqrT074BFp2qHsfl6ozW3JyfBSTrCzZmGbJEoypXuZL8N510GkwXD0dWtRvWpdPftrOog17+MPZvUiIifRTkMaYUGYJxlS14AmY+TvocTZcOQ1iWtZr98KSMv7ySRa92ydwaUanuncwxjRJVqZsDlOF/z4AX/0NjrsYLnwOwuvf+3hxwXq27D3Em9f/ysqSjWnGLMEYR3k5zLoDFr0AJ14DY/5e43T7tdm5r5Bn5mZzdt92DDkm2Q+BGmMaC0swxlnL5aObYdnbMOR/4awHap0RuTaPfbaK4rJy/jzaypKNae4swTR3pUUw7VpYOROG3wOn3X7EyeXnLfm8uziH60/tRnpyXAMHaoxpbCzBNGfFB+Dty53p9kdNgV/dcMSHUlUmz1xB6xZRTBphZcnGGEswzdehvfDGr2FLJlzwLxhw+VEd7tOft/P9+jweuvA4WlpZsjGGeiYYEQkD4lV1n5/iMYFQsNNZKCx3Jfx6KvQde1SHKywp4y+zsujVLoHfWFmyMcZV530wIvKmiLQUkThgBbBKRP7o/9CMX+zdDC+Pgt3ZcPk7R51cAF75ZgOb8w5xz5g+RITbrVXGGIcvvw36uj2WC4BPgM5A7csXmtC0KxteGun0YK7+ELqfcdSHzN1fxD//m82Zfdpyag9bgtoYc5gvCSbSXbb4AuAjVS3BWWXSNCbbf4aXR0JpIYyfCZ0HN8hhH/98FYUlZVaWbIypwpcE8xywAYgD5otIF8CuwTQmmxfBK6MhPMpZy6VD/wY57PKt+by9aDPXDEmnW0p8gxzTGNN01JlgVPUpVe2oqqPVsREYHoDYTENYOxdePR9atHHWcknp2SCHrZgtOSk2kltG9GiQYxpjmhZfLvLf6l7kFxF5UUR+AEYEIDZztFZ+DG9eCq26OGu5JHVusEN/tmIHC9fl8fuzepLYwsqSjTFV+TJEdq17kf9sIAWYADzi16jM0fvxHXjnKmh/PIz/GBLaNdihi0qd2ZJ7tI1n3KCGS1rGmKbFlwRTMW/IaOBlVf3R4zkTir5/AT6YCOlD4eqP6r2WS12mfrOBjbsPcu+YvlaWbIypkS+/HRaLyGc4CWa2iCQA5f4Nyxyxrx6HT26HnqPg8nchOqFBD7+roIh/fJHN8F4pnNbTypKNMTXz5U7+3wIDgHWqelBE2uAMk5lQogpz7oevn4Djf+1M/3IEa7nU5fHPV3OwpIy7z+3b4Mc2xjQtvlSRlQNpwD0i8hgwRFWX+XJwERkpIqtEJFtE7qzm9fEikisiS92v6zxemyIiy0UkS0SecosMEjzaLhWRXSLyhNv+9yKyQkSWicgXbjl181BeDh//3kkuGdfChc/7Jbms3L6Pt7/fxFWDu9C9rZUlG2NqV2cPRkQeAU4C3nCfukVEhqjqXXXsFw48DZwF5ACLRGS6qq7wavqOqk7y2ncIMBTo5z61ABimqvNwelMV7RYD77ubS4AMt5f1P8AU4Dd1nV+jV1YCH/4P/PQuDL0Vzvx/Rzzdfm0qypITYiK57UwrSzbG1M2XIbLRwAC3J4OITMX5ZV5rggEGAdmqus7d723gfJz5zOqiQAwQhVNQEAns8GwgIj2AtsBXAKo61+PlhcCVPrxP41ZSCNMmwKpP4Iz/g1N/77e3+iJrJ19n7+b+8/qS1CLKb+9jjGk6fC0BSvJ4nOjjPh2BzR7bOe5z3i52h7WmiUgnAFX9FpgLbHO/Zqtqltd+43B6P9VNW/NbYJaPcTZORfvhzV87yWX0Y35NLsWl5Tz0SRbHpMRxxeDmM/JojDk6vvRgHgaWiMhcnN7EadTde4HqS5m9k8EM4C1VLRKRG4GpwAgR6Q70wbn2A/C5iJymqvM99r2MaibdFJErgQxgWLVBiUwEJgJ07txI7+E4mOes5bJ1iXO9pb9/RwJSPiB7AAAag0lEQVRf/XYD63cd4OUJJxFpZcnGGB/5cpH/LWAwzrWO94GTgfm17uTIATwXB0kDtnode7eqFrmbLwAD3ccXAgtVtUBVC3B6I7/Mzigi/YEIVV3seTwRORO4GxjrcVzv83leVTNUNSMlpRGW2e7fAa+Mge3L4NJX/Z5c8g4U8+QXaxjWM4Xhvdr69b2MMU2LT3+Oquo2VZ2uqh+p6nacaxx1WQT0EJGuIhKF0+OY7tlARDp4bI4FKobBNgHDRCTCncl5mMdr4AyPveV1rBNwJuYcq6o7fTmvRmfvJmdG5D0b4PL/QJ8xfn/Lv3++moPFZdxzrs2WbIypnyNdMrnOMiVVLRWRScBsIBx4SVWXi8hkIFNVp+NUpI0FSoE8YLy7+zSc+c5+whlW+1RVZ3gc/lKc4gNPfwXigXfFqaLapKpHv5pWqNi1xpm0srjAWcul0yC/v+Wq7ft547uNXDW4Cz3aNewNm8aYpk+qv0Zex04im1S1kV7AOCwjI0MzMzODHUbdtv3oLHEsAld94Mwv5meqytUvfc+Pm/fy5R+H0yrOKseMMQ4RWayqGXW1q7EHIyL/oPqFxYTKVWXGnzYthDcudaZ8ufojSO4ekLedu2onX63Zxb1j+lpyMcYckdqGyGr7074R/NnfBGR/Ae9cCQkdnOSS1KnufRpASVk5D36cRbfkOK6ysmRjzBGqMcGo6tRABmK8rJgO7/0WknvBVe9DfOAquF5fuJF1uQd48ZoMoiKsLNkYc2Tst0coWvomvHsNdBgA42cENLnsOVDME3PWcGqPZEb0trJkY8yRswQTar57zplbrOtpzgX92FYBffsnv1jD/sIS7jm3L+KHOc2MMc3HkZYpm4amCvMfg7kPQu8xcPGLEBkT0BCyd+7ntYUbufxXnenV3sqSjTFHx5fZlJ+q5ul8nHtZPmr4kJohVfj8XvjmH9DvMjj/aQgPfO5/8OMsWkSF87szewb8vY0xTY8vQ2QxOFPkr3G/+gGtgd9WrMVijkJ5Gcy8zUkuJ13vLhQW+OQyd9VO5q3K5dYzetAmPjrg72+MaXp8+U3WHRihqqUAIvIv4DOcdV5+8mNsTV9ZCXxwA/z8Hpz6Bxhxr1/WcqlLSVk5D32cRXqbFlx9cnrA398Y0zT5kmA6AnE4w2K4j1NVtUxEqp1Q0vig5BD85xpYM9tZJOyU24IWypvfbSJ7ZwHPXzXQypKNMQ3GlwQzBVgqIvM4PF3/X0QkDpjjx9iarsJ98NY42Pg1nPs4nPTboIWSf7CEv89ZzZBj2nBW33ZBi8MY0/TUmWBU9UUR+QRnhUoB/qyqFdPu/9GfwTVJB/Pg9Yud+cUuegH6/Tqo4Tz5xRr2HSrh3jFWlmyMaVi+Xk0+CTjVfVyG17ouxkf7t8OrF0DeOrjsDeg1KqjhrM0t4NVvN/CbkzrTp0PLoMZijGl6fClTfgQnwbzhPnWLiAxRVV9WtTQV9mxwpts/sAuunObcSBlkf/k4i5jIcP5wtpUlG2Mani89mNHAAFUtBxCRqcASfFs22QDsXAmvXeBc2L/6I0irc5Zrv5u/OpcvVu7krlG9SbayZGOMH/haMuQ5PX+iPwJpsrYugZdHOfe7TPgkJJJLaVk5D368gs6tWzB+aHqwwzHGNFG+9GAeBpaIyFwOV5FZ78UXG7+BN38DMUnOKpRtjgl2RAC8tWgzq3cU8OyVA4mOCA92OMaYJsqXKrK33BLlk3ASzJ+wSTLrtmaOs5ZLYpqTXBLTgh0RAPmHSnj8s1UM7taac461smRjjP/4VEWmqtuA6RXbIrIJaPRLJvvN8g/gveuhbR9nRuS45GBH9It/fLGGvVaWbIwJgCPtidhvppr88BpMuxY6DoRrZoRUclmXW8Ar32zg0oGdODbVLqUZY/zrSBOMNmgUTcW3z8D0SdDtdGcVytikuvYIqL98spLoiDD+cI6VJRtj/K/GITIR+QfVJxKhclWZUYUvH4V5D0Of85y1XCJCq/T36+xdzMnawR0je9E2IbDrzBhjmqfarsFkHuFrzYsqzL4bFj4NA66A854KynT7tSkrVx6YuYK0VrFcO7RrsMMxxjQTNf4mVNWp3s+JSHtV3e7fkBqR8jKYcSsseQ1+dSOc8zCEhV6B3TuLNrNy+36eueJEYiKtLNkYExj1/W34iV+iaIxKi52L+Uteg9PugJGPhGRy2VdYwt8+W8Wg9NaMOq59sMMxxjQj9f2NWK/qMREZKSKrRCRbRO6s5vXxIpIrIkvdr+s8XpsiIstFJEtEnhJHgkfbpSKyq2JVTRE5TUR+EJFSEbmknudVP8UH4e3LYcWHcPaDMOLuoCwU5oun/5tN3sFiK0s2xgRcfS8WvOBrQxEJB57GWfkyB1gkItNVdYVX03dUdZLXvkOAoTjLMwMsAIap6jyc5Zsr2i0G3nc3NwHjgdt9jfGIffUYZM+B856EgeP9/nZHasOuA7z09XouOTGN49OsLNkYE1j1SjCq+kw9mg8CslV1HYCIvA2cD3gnmGrfCogBonB6TZHADs8GItIDaAt85ca2wX2+vB4xHplTb4cuQ6H7GX5/q6Px8KwsIsPD+OM5vYIdijGmGfLnRYOOwGaP7Rz3OW8Xi8gyEZkmIp0AVPVbYC6wzf2arapZXvuNw+n91OueHBGZKCKZIpKZm5tbn10Pi2oR8snlm7W7mL18BzedfgxtW1pZsjEm8PyZYKob8PdOBjOAdFXth7P88lQAEekO9AHScJLSCBHxXkDlMuCt+galqs+raoaqZqSkpNR390bBKUvOomNSLNed2i3Y4Rhjmil/JpgcoJPHdhpeK2Gq6m5VLXI3XwAGuo8vBBaqaoGqFgCzgMEV+4lIfyBCVRf7K/jG7N3MzWRt28edo3pbWbIxJmj8mWAWAT1EpKuIROH0OKZ7NhCRDh6bY4GKYbBNwDARiRCRSGCYx2vgDI/Vu/fSHOwvLOGxz1YzsEsrxvTrUPcOxhjjJ35LMKpaCkwCZuMkh/+o6nIRmSwiY91mt7ilyD8Ct+BUgQFMA9YCPwE/Aj+q6gyPw1+KV4IRkZNEJAf4NfCciCz306mFtGfmrWVXQRH3WVmyMSbIpJ7XyJuUjIwMzcxsOrPebM47yBl/+5Ix/Tvw+KUD6t7BGGOOgIgsVtU6l+cNvVvPzRF7eFYW4WHCHef0DnYoxhhjCaap+G7dbj75aTv/c/oxtE+0smRjTPBZgmkCysqVyTNXkJoYw/VWlmyMCRGWYJqA937IYfnWffxpVG9io6ws2RgTGizBNHIFRaX8dfYqTuicxNj+qcEOxxhjfmEJppH717xscvcX2WzJxpiQYwmmEducd5AXvlrPBQNSObFzq2CHY4wxlViCacQe/XQlYQJ3jLSyZGNM6LEE00hlbshj5rJt3HDaMaQmxQY7HGOMqcISTCNU7pYlt28Zww3DrCzZGBOaLME0Qh8s2cKynHz+NKoXLaLquyipMcYEhiWYRuZAUSlTZq+kf6ckzu9f3fptxhgTGizBNDLPfbmWHfuc2ZLDwqws2RgTuizBNCJb9h7iufnrGNs/lYFdrCzZGBPaLME0Io/OWgnAn0ZZWbIxJvRZgmkkFm/cw/QftzLxtG50tLJkY0wjYAmmESgvVx6YuYK2CdHcOOyYYIdjjDE+sQTTCEz/cStLN+/ljpG9iYu2smRjTONgCSbEHSwu5ZFZKzm+YyIXnWBlycaYxsMSTIh7fv46tu8r5L7zrCzZGNO4WIIJYdvyD/Hsl2s5t18HTkpvHexwjDGmXizBhLApn66iXOFOmy3ZGNMIWYIJUUs27eGDJVu4/tSudGrdItjhGGNMvVmCCUGqzmzJKQnR/M/p3YMdjjHGHBG/JhgRGSkiq0QkW0TurOb18SKSKyJL3a/rPF6bIiLLRSRLRJ4SR4JH26UisktEnnDbR4vIO+57fSci6f48N3+a/uNWlmzayx/P7kW8lSUbYxopv/32EpFw4GngLCAHWCQi01V1hVfTd1R1kte+Q4ChQD/3qQXAMFWdBwzwaLcYeN/d/C2wR1W7i8hlwKPAbxr2rPzvUHEZj85aybGpLbl4YFqwwzHGmCPmzx7MICBbVdepajHwNnC+j/sqEANEAdFAJLDDs4GI9ADaAl+5T50PTHUfTwPOEJFGV9f776/WsTW/kHvH9CXcypKNMY2YPxNMR2Czx3aO+5y3i0VkmYhME5FOAKr6LTAX2OZ+zVbVLK/9xuH0ftT7/VS1FMgH2jTUyQTCjn2FPDNvLaOOa8/gbo0qdGOMqcKfCaa6P7/Va3sGkK6q/YA5uD0QEekO9AHScBLHCBE5zWvfy4C36vl+iMhEEckUkczc3FyfTiRQpny6irJy5a5RfYIdijHGHDV/JpgcoJPHdhqw1bOBqu5W1SJ38wVgoPv4QmChqhaoagEwCxhcsZ+I9AciVHVxde8nIhFAIpDnHZSqPq+qGaqakZKScjTn16CW5ezlvR9yuPaUrnRuY2XJxpjGz58JZhHQQ0S6ikgUTo9jumcDEengsTkWqBgG2wQME5EIEYkEhnm8Bs7wmGfvBffY17iPLwH+6zF8FtJUlckzVpAcH8XNw222ZGNM0+C3KjJVLRWRScBsIBx4SVWXi8hkIFNVpwO3iMhYoBSntzHe3X0aMAL4CWeY61NVneFx+EuB0V5v+SLwmohku8e6zD9n1vA+/mkbmRv38MhFx5MQExnscIwxpkFII/kj3y8yMjI0MzMzqDEUlpRxxt++pGVsJDP/9xSrHDPGhDwRWayqGXW1szv5g+zFBevZsvcQ947pY8nFGNOkWIIJop37Cnl6bjZn923HkGOSgx2OMcY0KEswQfTX2asoKSvnz6OtLNkY0/RYggmSn7fkM+2HHCYM7Up6clywwzHGmAZnCSYIKmZLbt0iikkjbLZkY0zTZAkmCD79eTvfr8/j92f3pKWVJRtjmihLMAFWWFLGX2Zl0bt9Ar/J6FT3DsYY00hZggmwl7/ewOa8Q9w7pi8R4fbjN8Y0XfYbLoB27nfKks/s046h3a0s2RjTtFmCCaDHP1tNUWkZd59rZcnGmKbPEkyALN+azzuZm7n65HS6WlmyMaYZsAQTABWzJSfFRnLLiB7BDscYYwLCEkwAzF6+g+/W5/H7s3qS2MLKko0xzYMlGD8rKi3j4VlZ9Ggbz7hBnYMdjjHGBIwlGD+b+s0GNu4+aGXJxphmx37j+dGugiL+8UU2I3q35bSeobM8szHGBIIlGD96/PPVHCops9mSjTHNkiUYP8nato+3v9/EVSd3oXvb+GCHY4wxAWcJxg9UlQc/XkHL2EhuPcPKko0xzZMlGD+Yk7WTr7N3c9sZPUhqERXscIwxJigswTSw4tJyHvp4BcekxHHF4C7BDscYY4LGEkwDe/XbDWzYfZB7xvQl0sqSjTHNmP0GbEC7C4p48os1DOuZwvBebYMdjjHGBJUlmAb0xJw1HCwu4x6bLdkYY/ybYERkpIisEpFsEbmzmtfHi0iuiCx1v67zeG2KiCwXkSwReUpExH0+SkSeF5HVIrJSRC52n+8iIl+IyDIRmSciaf48N2+rtu/nje82cuWvOtOjXUIg39oYY0JShL8OLCLhwNPAWUAOsEhEpqvqCq+m76jqJK99hwBDgX7uUwuAYcA84G5gp6r2FJEwoLXb5jHgVVWdKiIjgIeBqxr+zKqqKEuOj47gtjN7BuItjTEm5PmzBzMIyFbVdapaDLwNnO/jvgrEAFFANBAJ7HBfuxYneaCq5aq6y32+L/CF+3huPd7rqM1dtZOv1uzitjN70irOypKNMQb8m2A6Aps9tnPc57xd7A5rTRORTgCq+i1Oktjmfs1W1SwRSXL3eUBEfhCRd0Wknfvcj8DF7uMLgQQRadPA51RFSVk5D87MoltKHFedbGXJxhhTwZ8JRqp5Tr22ZwDpqtoPmANMBRCR7kAfIA0nKY0QkdNwhvTSgK9V9UTgW5yhMYDbgWEisgRnOG0LUFolKJGJIpIpIpm5ublHeYrw2rcbWbfrAPec28fKko0xxoM/fyPmAJ08ttOArZ4NVHW3qha5my8AA93HFwILVbVAVQuAWcBgYDdwEPjAbfcucKJ7rK2qepGqnoBznQZVzfcOSlWfV9UMVc1ISTm6GY73HCjmiTmrObVHspUlG2OMF38mmEVADxHpKiJRwGXAdM8GItLBY3MskOU+3oTTG4kQkUicHkmWqipOr+d0t90ZwAr3WMnuRX+Au4CXGv6UKntizmoKikq559y+uEVuxhhjXH6rIlPVUhGZBMwGwoGXVHW5iEwGMlV1OnCLiIzFGcrKA8a7u08DRgA/4QyrfaqqM9zX/gS8JiJPALnABPf504GHRUSB+cDN/jo3gDU79vP6d5u4/Fed6dXeypKNMcabOJ2C5ikjI0MzMzOPaN9rXvqeHzbtYd7tp9MmPrqBIzPGmNAlIotVNaOudnZV+gjMXbWTL1fncusZPSy5GGNMDSzBHIGCwlJO6JzE1SenBzsUY4wJWX67BtOUndc/lTH9OtiFfWOMqYX1YI6QJRdjjKmdJRhjjDF+YQnGGGOMX1iCMcYY4xeWYIwxxviFJRhjjDF+YQnGGGOMX1iCMcYY4xfNei4yEckFNgY7Dg/JwK46WwVPqMcHoR9jqMcHoR9jqMcHTT/GLqpa53onzTrBhBoRyfRlArlgCfX4IPRjDPX4IPRjDPX4wGKsYENkxhhj/MISjDHGGL+wBBNang92AHUI9fgg9GMM9fgg9GMM9fjAYgTsGowxxhg/sR6MMcYYv7AEE2AiMlJEVolItojcWUObS0VkhYgsF5E3Qy1GEeksInNFZImILBOR0QGO7yUR2SkiP9fwuojIU278y0TkxBCL7wo3rmUi8o2I9A9kfL7E6NHuJBEpE5FLAhWbx3vXGaOInC4iS93PypehFJ+IJIrIDBH50Y1vQoDj6+R+TrPc97+1mjb+/ayoqn0F6AsIB9YC3YAo4Eegr1ebHsASoJW73TYEY3we+B/3cV9gQ4BjPA04Efi5htdHA7MAAQYD34VYfEM8/n1HBTo+X2L0+L/wX+AT4JJQixFIAlYAnd3tQH9W6orvz8Cj7uMUIA+ICmB8HYAT3ccJwOpqPst+/axYDyawBgHZqrpOVYuBt4HzvdpcDzytqnsAVHVnCMaoQEv3cSKwNYDxoarzcT6sNTkfeFUdC4EkEekQmOjqjk9Vv6n49wUWAmkBCaxyDHX9DAH+F3gPCPT/QcCnGC8H3lfVTW77gMbpQ3wKJIizOmG827Y0ELEBqOo2Vf3BfbwfyAI6ejXz62fFEkxgdQQ2e2znUPUfvCfQU0S+FpGFIjIyYNE5fInxfuBKEcnB+ev2fwMTms98OYdQ8VucvyBDioh0BC4Eng12LLXoCbQSkXkislhErg52QF7+CfTB+QPsJ+BWVS0PRiAikg6cAHzn9ZJfPysRDXUg45Pq1ln2LuOLwBkmOx3nL9uvROQ4Vd3r59gq+BLjOOAVVf2biJwMvObGGJQPTzV8OYegE5HhOAnmlGDHUo0ngD+palkILw8eAQwEzgBigW9FZKGqrg5uWL84B1gKjACOAT4Xka9UdV8ggxCReJye6G3VvLdfPyuWYAIrB+jksZ1G1eGlHGChqpYA60VkFU7CWRSYEH2K8bfASABV/VZEYnDmNQrKUEo1fDmHoBKRfsC/gVGqujvY8VQjA3jbTS7JwGgRKVXVD4MbViU5wC5VPQAcEJH5QH+caw2hYALwiDoXO7JFZD3QG/g+UAGISCROcnlDVd+vpolfPys2RBZYi4AeItJVRKKAy4DpXm0+BIYDiEgyzjDAuhCLcRPOX42ISB8gBsgNYIx1mQ5c7VbIDAbyVXVbsIOqICKdgfeBq0Lor+1KVLWrqqarajowDbgpxJILwEfAqSISISItgF/hXGcIFZ6fk3ZALwL4WXav/bwIZKnq4zU08+tnxXowAaSqpSIyCZiNU6HzkqouF5HJQKaqTndfO1tEVgBlwB8D+ReujzH+AXhBRH6H050e7/6VFhAi8hbOEGKyex3o/4BIN/5nca4LjQaygYM4f0kGjA/x3Qe0AZ5xewilGuCJEX2IMejqilFVs0TkU2AZUA78W1VrLbsOZHzAA8ArIvITzlDUn1Q1kDMsDwWuAn4SkaXuc38GOnvE6NfPit3Jb4wxxi9siMwYY4xfWIIxxhjjF5ZgjDHG+IUlGGOMMX5hCcYYY4xfWIIxxhjjF5ZgjGkE3GnpZx5tG2MCyRKMMQ3AvRPaPk/GeLAPhDFHSETS3cWcngF+AK4SkW9F5AcRededZBARGS0iK0Vkgbu4U429DBEZJM4iZEvc772qaXO/iLwmIv8VkTUicr3Hy/EiMs19vzfc6UIQkftEZJGI/Cwiz1c8b4w/WYIx5uj0Al4FzsKZBPRMVT0RyAR+704E+hzOpJan4Cw8VZuVwGmqegLOlDJ/qaFdP+Bc4GTgPhFJdZ8/AbgNZyG4bjjThQD8U1VPUtXjcGYeHlPvMzWmnmwuMmOOzkZVXSgiY3B+qX/tdg6igG9xZs9dp6rr3fZvARNrOV4iMFVEeuDM8xZZQ7uPVPUQcEhE5uIsFLcX+F5VcwDc+afSgQXAcBG5A2gBtAaWAzOO7JSN8Y0lGGOOzgH3uwCfq+o4zxdF5IR6Hu8BYK6qXuguEjWvhnbekwhWbBd5PFcGRLi9qGeADFXdLCL348yAbYxf2RCZMQ1jITBURLoDiEgLEemJM+TVzU0WAL+p4ziJwBb38fha2p0vIjEi0gZnRt/a1guqSCa73OtCl9QRgzENwhKMMQ1AVXNxEsJbIrIMJ+H0doexbgI+FZEFwA4gv5ZDTQEeFpGvcZZLqMn3wMfu+zygqjUuEuWuhvoCzrK9HxK4xetMM2fT9RvjZyISr6oFbuXW08AaVf37URzvfqBAVR9rqBiN8QfrwRjjf9e7F9yX4wyBPRfkeIwJCOvBGBMEIjIBuNXr6a9V9eZgxGOMP1iCMcYY4xc2RGaMMcYvLMEYY4zxC0swxhhj/MISjDHGGL+wBGOMMcYv/j++MJzNsQm9UQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f761a1ee208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "利用训练好的模型，在测试集上进行测试"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.0         1   2950      1475.000000     1475.000000        0.0   \n",
       "1        1.0         2   2850      1425.000000      950.000000       -1.0   \n",
       "2        1.0         1   3758      1879.000000     1879.000000        0.0   \n",
       "3        1.0         2   3300      1650.000000     1100.000000       -1.0   \n",
       "4        2.0         2   4900      1633.333333     1633.333333        0.0   \n",
       "\n",
       "   room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       "0       2.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "1       3.0  2016      6   24  ...         0     0      0    1       0      0   \n",
       "2       2.0  2016      6    3  ...         0     0      0    0       0      0   \n",
       "3       3.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "4       4.0  2016      4   12  ...         0     0      0    1       0      0   \n",
       "\n",
       "   wheelchair  wifi  windows  work  \n",
       "0           0     0        0     0  \n",
       "1           0     0        0     0  \n",
       "2           0     0        0     0  \n",
       "3           1     0        0     0  \n",
       "4           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "#dpath = './data/'\n",
    "X_test = pd.read_csv(\"RentListingInquries_FE_test.csv\")\n",
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#xgb5_1\n",
    "\n",
    "bst = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=270,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=4,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        reg_alpha= 1.5, \n",
    "        reg_lambda= 0.5,\n",
    "        seed=3)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.9, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=4, min_child_weight=1, missing=None, n_estimators=270,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=1.5, reg_lambda=0.5, scale_pos_weight=1, seed=3,\n",
       "       silent=True, subsample=0.8)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bst.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuary: 76.59%\n"
     ]
    }
   ],
   "source": [
    "train_preds = bst.predict(X_train)\n",
    "train_predictions = [round(value) for value in train_preds]\n",
    "#y_train = dtrain.get_label()\n",
    "train_accuracy = accuracy_score(y_train, train_predictions)\n",
    "print (\"Train Accuary: %.2f%%\" % (train_accuracy * 100.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "看起来参数还有待调整，不够完美"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jck/.local/lib/python3.6/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "# make prediction\n",
    "X_test = np.array(X_test)\n",
    "preds = bst.predict(X_test)\n",
    "predictions = [round(value) for value in preds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_pred = bst.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df1 = pd.DataFrame(y_test_pred)\n",
    "out_df1.columns = [\"high\", \"medium\", \"low\"]\n",
    "\n",
    "out_df = pd.concat([out_df1], axis = 1)\n",
    "out_df.to_csv(\"xgb_Rent.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
